@cryptotaxi247 / CoPilot / commits / 2bb57ddd

full, wazuh, and velociraptor healthchecks (#38)

* full, wazuh, and velociraptor healthchecks full healthcheck determines the status of every agents wazuh, and velo services. if agent wazuh is healthy, check the wazuh-indexer to determine if logs have been recently collected (1, 5, 15) minute intervals * precommit fixes

taylor_socfortress committed Jul 16, 2023 at 11:09 UTC 2bb57ddd41dc4b60b019f08f5bbd86b5d888e34f
6 files changed +637 -1
backend/app/__init__.py
+2
@@ -50,6 +50,7 @@ from app.routes.alerts import bp as alerts_bp
50 from app.routes.connectors import bp as connectors_bp
51 from app.routes.dfir_iris import bp as dfir_iris_bp
52 from app.routes.graylog import bp as graylog_bp
53 +from app.routes.healthchecks import bp as healthchecks_bp
54 from app.routes.influxdb import bp as influxdb_bp
55 from app.routes.rules import bp as rules_bp
56 from app.routes.shuffle import bp as shuffle_bp
@@ -70,3 +71,4 @@ app.register_blueprint(dfir_iris_bp) # Register the dfir_iris blueprint
71 app.register_blueprint(sublime_bp) # Register the sublime blueprint
72 app.register_blueprint(influxdb_bp) # Register the influxdb blueprint
73 app.register_blueprint(smtp_bp) # Register the smtp blueprint
74 +app.register_blueprint(healthchecks_bp) # Register the healthchecks blueprint
backend/app/routes/healthchecks.py new
+117
@@ -0,0 +1,117 @@
1 +from typing import Any
2 +
3 +from flask import Blueprint
4 +from flask import jsonify
5 +from loguru import logger
6 +
7 +from app.models.agents import agent_metadata_schema
8 +from app.services.agents.agents import AgentService
9 +from app.services.Healthchecks.agents import HealthcheckAgentsService
10 +
11 +bp = Blueprint("healthchecks", __name__)
12 +
13 +
14 +@bp.route("/healthcheck/agent/full", methods=["GET"])
15 +def get_agents_full() -> Any:
16 + """
17 + Endpoint to get a list of all agents who have sent logs within the last 15 minutes. Also returns a list of agents who have not sent
18 + logs within the last 15 minutes.
19 + Returns:
20 + json: A JSON response containing the list of all available agents along with their log existence status.
21 + """
22 + agent_service = AgentService()
23 + agents = agent_service.get_all_agents()
24 + healthcheck_service = HealthcheckAgentsService()
25 + agent_health = healthcheck_service.perform_healthcheck_full(agents, check_logs=True)
26 + return jsonify(agent_health)
27 +
28 +
29 +@bp.route("/healthcheck/agent/<agent_id>/full", methods=["GET"])
30 +def get_agent_full(agent_id: str) -> Any:
31 + """
32 + Endpoint to get the log existence status of a specific agent.
33 + Args:
34 + agent_id (str): The ID of the agent.
35 + Returns:
36 + json: A JSON response containing the log existence status of the agent.
37 + """
38 + # Query the `agent_metadata` table for the agent to get the hostname
39 + agent_service = AgentService()
40 + healthcheck_service = HealthcheckAgentsService()
41 + agent = agent_service.get_agent(agent_id=agent_id)
42 + if agent is None:
43 + return jsonify({"success": False, "message": "Agent not found."}), 404
44 + agent = agent_metadata_schema.dump(agent)
45 + agent_health = healthcheck_service.perform_healthcheck_full([agent], check_logs=True)
46 + return jsonify(agent_health)
47 +
48 +
49 +@bp.route("/healthcheck/agent/wazuh", methods=["GET"])
50 +def get_agents_wazuh() -> Any:
51 + """
52 + Endpoint to get a list of all agents whose Wazuh-Agent is running. Also returns a list of agents whose Wazuh-Agent is not running.
53 + Returns:
54 + json: A JSON response containing the list of all available agents along with their Wazuh-Agent status.
55 + """
56 + agent_service = AgentService()
57 + agents = agent_service.get_all_agents()
58 + healthcheck_service = HealthcheckAgentsService()
59 + agent_health = healthcheck_service.perform_healthcheck_wazuh(agents)
60 + return jsonify(agent_health)
61 +
62 +
63 +@bp.route("/healthcheck/agent/<agent_id>/wazuh", methods=["GET"])
64 +def get_agent_wazuh(agent_id: str) -> Any:
65 + """
66 + Endpoint to get the Wazuh-Agent status of a specific agent.
67 + Args:
68 + agent_id (str): The ID of the agent.
69 + Returns:
70 + json: A JSON response containing the Wazuh-Agent status of the agent.
71 + """
72 + # Query the `agent_metadata` table for the agent to get the hostname
73 + agent_service = AgentService()
74 + healthcheck_service = HealthcheckAgentsService()
75 + agent = agent_service.get_agent(agent_id=agent_id)
76 + if agent is None:
77 + return jsonify({"success": False, "message": "Agent not found."}), 404
78 + agent = agent_metadata_schema.dump(agent)
79 + logger.info(f"Checking Wazuh-Agent status for agent {agent}.")
80 + health = healthcheck_service.perform_healthcheck_wazuh(agent)
81 + return jsonify(health)
82 +
83 +
84 +@bp.route("/healthcheck/agent/velociraptor", methods=["GET"])
85 +def get_agents_velociraptor() -> Any:
86 + """
87 + Endpoint to get a list of all agents whose Velociraptor service is running. Also returns a list of agents whose Velociraptor
88 + service is not running.
89 + Returns:
90 + json: A JSON response containing the list of all available agents along with their Velociraptor service status.
91 + """
92 + agent_service = AgentService()
93 + agents = agent_service.get_all_agents()
94 + healthcheck_service = HealthcheckAgentsService()
95 + agent_health = healthcheck_service.perform_healthcheck_velociraptor(agents)
96 + return jsonify(agent_health)
97 +
98 +
99 +@bp.route("/healthcheck/agent/<agent_id>/velociraptor", methods=["GET"])
100 +def get_agent_velociraptor(agent_id: str) -> Any:
101 + """
102 + Endpoint to get the Velociraptor service status of a specific agent.
103 + Args:
104 + agent_id (str): The ID of the agent.
105 + Returns:
106 + json: A JSON response containing the Velociraptor service status of the agent.
107 + """
108 + # Query the `agent_metadata` table for the agent to get the hostname
109 + agent_service = AgentService()
110 + healthcheck_service = HealthcheckAgentsService()
111 + agent = agent_service.get_agent(agent_id=agent_id)
112 + if agent is None:
113 + return jsonify({"success": False, "message": "Agent not found."}), 404
114 + agent = agent_metadata_schema.dump(agent)
115 + logger.info(f"Checking Velociraptor service status for agent {agent}.")
116 + health = healthcheck_service.perform_healthcheck_velociraptor(agent)
117 + return jsonify(health)
backend/app/services/Healthchecks/agents.py new
+191
@@ -0,0 +1,191 @@
1 +from datetime import datetime
2 +from datetime import timedelta
3 +from typing import Dict
4 +from typing import List
5 +from typing import Union
6 +
7 +from loguru import logger
8 +
9 +from app.services.WazuhIndexer.universal import UniversalService
10 +
11 +
12 +class HealthcheckAgentsService:
13 + """
14 + A service class that encapsulates the logic for CoPilot healthchecks.
15 + """
16 +
17 + SKIP_INDEX_NAMES: Dict[str, bool] = {
18 + "wazuh-statistics": True,
19 + "wazuh-monitoring": True,
20 + }
21 +
22 + def __init__(self):
23 + self.universal_service = UniversalService()
24 +
25 + def convert_string_to_datetime(self, date_string: str) -> datetime:
26 + try:
27 + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S")
28 + except ValueError:
29 + return datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%f")
30 +
31 + def is_agent_unhealthy(self, agent: Dict, current_time: datetime) -> bool:
32 + last_seen = self.convert_string_to_datetime(agent["last_seen"])
33 + client_last_seen = self.convert_string_to_datetime(agent["client_last_seen"])
34 +
35 + agent["unhealthy_wazuh_agent"] = (current_time - last_seen) > timedelta(minutes=30)
36 + agent["unhealthy_velociraptor_client"] = (current_time - client_last_seen) > timedelta(minutes=30)
37 +
38 + return agent["unhealthy_wazuh_agent"] or agent["unhealthy_velociraptor_client"]
39 +
40 + def get_indices(self):
41 + """
42 + Returns a list of all indices in the Wazuh-Indexer.
43 + """
44 + indices_response = self.universal_service.collect_indices()
45 + if indices_response["success"]:
46 + return indices_response["indices_list"]
47 + else:
48 + logger.error("Failed to collect indices.")
49 + return []
50 +
51 + def has_agent_recent_logs(self, agent: Dict, indices: List[str]) -> bool:
52 + for interval in [1, 5, 15]:
53 + logger.info(f"Checking agent {agent['hostname']} for logs within the last {interval} minutes.")
54 + query = self._generate_recent_logs_query(agent["hostname"], interval)
55 + for index in indices:
56 + if any(skip_index in index for skip_index in self.SKIP_INDEX_NAMES):
57 + continue
58 + response = self.universal_service.run_query(query, index, size=1)
59 + if response["query_results"]["hits"]["total"]["value"] > 0:
60 + return True
61 + # Check the next interval only if the previous interval didn't return results
62 + if interval == 1:
63 + break
64 + # Proceed to next agent if logs are found
65 + break
66 + return False
67 +
68 + @staticmethod
69 + def _generate_recent_logs_query(agent_hostname: str, minutes: int) -> Dict:
70 + return {
71 + "query": {"bool": {"must": [{"match": {"agent_name": agent_hostname}}, {"range": {"timestamp": {"gte": f"now-{minutes}m"}}}]}},
72 + }
73 +
74 + def perform_healthcheck_full(self, agents: List[Dict], check_logs: bool = False) -> Dict:
75 + """
76 + Checks the health of all agents.
77 + Args:
78 + agents (list): A list of all agents.
79 + check_logs (bool): Whether to check if agents have recent logs.
80 + Returns:
81 + healthy_agents: A list of all agents with a healthy status.
82 + unhealthy_agents: A list of all agents with an unhealthy status.
83 + """
84 + current_time = datetime.now()
85 + healthy_wazuh_agents = []
86 + unhealthy_wazuh_agents = []
87 + healthy_velociraptor_agents = []
88 + unhealthy_velociraptor_agents = []
89 + healthy_recent_logs_collected = []
90 + unhealthy_recent_logs_collected = []
91 +
92 + # Get the indices only once before the loop
93 + indices = self.get_indices()
94 +
95 + for agent in agents:
96 + self.is_agent_unhealthy(agent, current_time)
97 +
98 + # Wazuh agents
99 + if agent["unhealthy_wazuh_agent"]:
100 + unhealthy_wazuh_agents.append(agent)
101 + else:
102 + healthy_wazuh_agents.append(agent)
103 +
104 + # Velociraptor clients
105 + if agent["unhealthy_velociraptor_client"]:
106 + unhealthy_velociraptor_agents.append(agent)
107 + else:
108 + healthy_velociraptor_agents.append(agent)
109 +
110 + # Recent logs check
111 + if check_logs:
112 + has_recent_logs = self.has_agent_recent_logs(agent, indices)
113 + if has_recent_logs:
114 + healthy_recent_logs_collected.append(agent)
115 + else:
116 + unhealthy_recent_logs_collected.append(agent)
117 +
118 + return {
119 + "healthy_wazuh_agents": healthy_wazuh_agents,
120 + "unhealthy_wazuh_agents": unhealthy_wazuh_agents,
121 + "healthy_velociraptor_agents": healthy_velociraptor_agents,
122 + "unhealthy_velociraptor_agents": unhealthy_velociraptor_agents,
123 + "healthy_recent_logs_collected": healthy_recent_logs_collected,
124 + "unhealthy_recent_logs_collected": unhealthy_recent_logs_collected,
125 + "message": "Successfully retrieved agent healthcheck.",
126 + "success": True,
127 + }
128 +
129 + def perform_healthcheck_wazuh(self, agents: Union[List[Dict], Dict]) -> Dict:
130 + """
131 + Checks the health of Wazuh agents.
132 + Args:
133 + agents (list or dict): Either a list of agents or a single agent.
134 + Returns:
135 + healthy_agents: A list of all agents with a healthy status.
136 + unhealthy_agents: A list of all agents with an unhealthy status.
137 + """
138 + current_time = datetime.now()
139 + healthy_wazuh_agents = []
140 + unhealthy_wazuh_agents = []
141 +
142 + if isinstance(agents, dict): # If a single agent is provided
143 + agents = [agents] # Convert the single agent to a list
144 +
145 + for agent in agents:
146 + self.is_agent_unhealthy(agent, current_time)
147 +
148 + # Wazuh agents
149 + if agent["unhealthy_wazuh_agent"]:
150 + unhealthy_wazuh_agents.append(agent)
151 + else:
152 + healthy_wazuh_agents.append(agent)
153 +
154 + return {
155 + "healthy_wazuh_agents": healthy_wazuh_agents,
156 + "unhealthy_wazuh_agents": unhealthy_wazuh_agents,
157 + "message": "Successfully retrieved wazuh agent healthcheck.",
158 + "success": True,
159 + }
160 +
161 + def perform_healthcheck_velociraptor(self, agents: Union[List[Dict], Dict]) -> Dict:
162 + """
163 + Checks the health of Velociraptor clients.
164 + Args:
165 + agents (list or dict): Either a list of agents or a single agent.
166 + Returns:
167 + healthy_agents: A list of all agents with a healthy status.
168 + unhealthy_agents: A list of all agents with an unhealthy status.
169 + """
170 + current_time = datetime.now()
171 + healthy_velociraptor_agents = []
172 + unhealthy_velociraptor_agents = []
173 +
174 + if isinstance(agents, dict):
175 + agents = [agents]
176 +
177 + for agent in agents:
178 + self.is_agent_unhealthy(agent, current_time)
179 +
180 + # Velociraptor clients
181 + if agent["unhealthy_velociraptor_client"]:
182 + unhealthy_velociraptor_agents.append(agent)
183 + else:
184 + healthy_velociraptor_agents.append(agent)
185 +
186 + return {
187 + "healthy_velociraptor_agents": healthy_velociraptor_agents,
188 + "unhealthy_velociraptor_agents": unhealthy_velociraptor_agents,
189 + "message": "Successfully retrieved velociraptor client healthcheck.",
190 + "success": True,
191 + }
backend/app/services/WazuhIndexer/universal.py
+52
@@ -98,3 +98,55 @@ class UniversalService:
98 except Exception as e:
99 logger.error(f"Failed to collect indices: {e}")
100 return {"message": "Failed to collect indices", "success": False}
101 +
102 + def run_query(self, query: str, index: str, size: int = 10000):
103 + """
104 + Runs a query against the Wazuh-Indexer.
105 +
106 + Args:
107 + query (str): The query to run against the Wazuh-Indexer.
108 + index (str): The index to run the query against.
109 + size (int): The number of results to return.
110 +
111 + Returns:
112 + dict: A dictionary containing the results of the query.
113 + """
114 + if self.connector_url is None or self.connector_username is None or self.connector_password is None:
115 + return {
116 + "message": "Failed to collect Wazuh-Indexer details",
117 + "success": False,
118 + }
119 +
120 + query_results = self._run_query(query, index, size)
121 +
122 + if query_results["success"]:
123 + return query_results
124 +
125 + return {"message": "Failed to run query", "success": False}
126 +
127 + def _run_query(self, query: str, index: str, size: int = 10000):
128 + """
129 + Wazuh-Indexer query to run a query against the Wazuh-Indexer.
130 +
131 + Args:
132 + query (str): The query to run against the Wazuh-Indexer.
133 + index (str): The index to run the query against.
134 + size (int): The number of results to return.
135 +
136 + Returns:
137 + dict: A dictionary containing the results of the query.
138 + """
139 + try:
140 + query_results = self.es.search(
141 + index=index,
142 + body=query,
143 + size=size,
144 + )
145 + return {
146 + "message": "Successfully ran query",
147 + "success": True,
148 + "query_results": query_results,
149 + }
150 + except Exception as e:
151 + logger.error(f"Failed to run query: {e}")
152 + return {"message": "Failed to run query", "success": False}
backend/app/services/agents/agents.py
-1
@@ -1,4 +1,3 @@
1 -# Here is the improved version of the code:
1 from datetime import datetime
2 from typing import Dict
3 from typing import List
backend/app/static/swagger.json
+275
@@ -90,6 +90,14 @@
90 "url": "http://swagger.io"
91 }
92 },
93 + {
94 + "name": "Healthcheck",
95 + "description": "Everything about Healthcheck",
96 + "externalDocs": {
97 + "description": "Find out more",
98 + "url": "http://swagger.io"
99 + }
100 + },
101 {
102 "name": "SMTP",
103 "description": "Everything about SMTP",
@@ -2735,6 +2743,273 @@
2743 }
2744 }
2745 }
2746 + },
2747 + "/healthcheck/agent/full": {
2748 + "get": {
2749 + "summary": "Get full agent healthcheck",
2750 + "description": "Endpoint to get full agent healthcheck.",
2751 + "responses": {
2752 + "200": {
2753 + "description": "Successful operation",
2754 + "content": {
2755 + "application/json": {
2756 + "schema": {
2757 + "type": "object",
2758 + "properties": {
2759 + "agent_healthcheck": {
2760 + "type": "array",
2761 + "items": {
2762 + "type": "object",
2763 + "description": "Agent healthcheck details"
2764 + }
2765 + }
2766 + }
2767 + }
2768 + }
2769 + }
2770 + },
2771 + "default": {
2772 + "description": "Unexpected error",
2773 + "content": {
2774 + "application/json": {
2775 + "schema": {
2776 + "$ref": "#/components/schemas/Error"
2777 + }
2778 + }
2779 + }
2780 + }
2781 + },
2782 + "operationId": "getFullAgentHealthcheck",
2783 + "tags": ["Healthcheck"]
2784 + }
2785 + },
2786 + "/healthcheck/agent/{agent_id}/full": {
2787 + "get": {
2788 + "summary": "Get full agent healthcheck by agent ID",
2789 + "description": "Endpoint to get full agent healthcheck by agent ID.",
2790 + "parameters": [
2791 + {
2792 + "name": "agent_id",
2793 + "in": "path",
2794 + "description": "The ID of the agent.",
2795 + "required": true,
2796 + "schema": {
2797 + "type": "string"
2798 + }
2799 + }
2800 + ],
2801 + "responses": {
2802 + "200": {
2803 + "description": "Successful operation",
2804 + "content": {
2805 + "application/json": {
2806 + "schema": {
2807 + "type": "object",
2808 + "properties": {
2809 + "agent_healthcheck": {
2810 + "type": "array",
2811 + "items": {
2812 + "type": "object",
2813 + "description": "Agent healthcheck details"
2814 + }
2815 + }
2816 + }
2817 + }
2818 + }
2819 + }
2820 + },
2821 + "default": {
2822 + "description": "Unexpected error",
2823 + "content": {
2824 + "application/json": {
2825 + "schema": {
2826 + "$ref": "#/components/schemas/Error"
2827 + }
2828 + }
2829 + }
2830 + }
2831 + },
2832 + "operationId": "getFullAgentHealthcheckByAgentID",
2833 + "tags": ["Healthcheck"]
2834 + }
2835 + },
2836 + "/healthcheck/agent/wazuh": {
2837 + "get": {
2838 + "summary": "Get Wazuh agent healthcheck",
2839 + "description": "Endpoint to get Wazuh agent healthcheck.",
2840 + "responses": {
2841 + "200": {
2842 + "description": "Successful operation",
2843 + "content": {
2844 + "application/json": {
2845 + "schema": {
2846 + "type": "object",
2847 + "properties": {
2848 + "agent_healthcheck": {
2849 + "type": "array",
2850 + "items": {
2851 + "type": "object",
2852 + "description": "Agent healthcheck details"
2853 + }
2854 + }
2855 + }
2856 + }
2857 + }
2858 + }
2859 + },
2860 + "default": {
2861 + "description": "Unexpected error",
2862 + "content": {
2863 + "application/json": {
2864 + "schema": {
2865 + "$ref": "#/components/schemas/Error"
2866 + }
2867 + }
2868 + }
2869 + }
2870 + },
2871 + "operationId": "getWazuhAgentHealthcheck",
2872 + "tags": ["Healthcheck"]
2873 + }
2874 + },
2875 + "/healthcheck/agent/{agent_id}/wazuh": {
2876 + "get": {
2877 + "summary": "Get Wazuh agent healthcheck by agent ID",
2878 + "description": "Endpoint to get Wazuh agent healthcheck by agent ID.",
2879 + "parameters": [
2880 + {
2881 + "name": "agent_id",
2882 + "in": "path",
2883 + "description": "The ID of the agent.",
2884 + "required": true,
2885 + "schema": {
2886 + "type": "string"
2887 + }
2888 + }
2889 + ],
2890 + "responses": {
2891 + "200": {
2892 + "description": "Successful operation",
2893 + "content": {
2894 + "application/json": {
2895 + "schema": {
2896 + "type": "object",
2897 + "properties": {
2898 + "agent_healthcheck": {
2899 + "type": "array",
2900 + "items": {
2901 + "type": "object",
2902 + "description": "Agent healthcheck details"
2903 + }
2904 + }
2905 + }
2906 + }
2907 + }
2908 + }
2909 + },
2910 + "default": {
2911 + "description": "Unexpected error",
2912 + "content": {
2913 + "application/json": {
2914 + "schema": {
2915 + "$ref": "#/components/schemas/Error"
2916 + }
2917 + }
2918 + }
2919 + }
2920 + },
2921 + "operationId": "getWazuhAgentHealthcheckByAgentID",
2922 + "tags": ["Healthcheck"]
2923 + }
2924 + },
2925 + "/healthcheck/agent/velociraptor": {
2926 + "get": {
2927 + "summary": "Get Velociraptor agent healthcheck",
2928 + "description": "Endpoint to get Velociraptor agent healthcheck.",
2929 + "responses": {
2930 + "200": {
2931 + "description": "Successful operation",
2932 + "content": {
2933 + "application/json": {
2934 + "schema": {
2935 + "type": "object",
2936 + "properties": {
2937 + "agent_healthcheck": {
2938 + "type": "array",
2939 + "items": {
2940 + "type": "object",
2941 + "description": "Agent healthcheck details"
2942 + }
2943 + }
2944 + }
2945 + }
2946 + }
2947 + }
2948 + },
2949 + "default": {
2950 + "description": "Unexpected error",
2951 + "content": {
2952 + "application/json": {
2953 + "schema": {
2954 + "$ref": "#/components/schemas/Error"
2955 + }
2956 + }
2957 + }
2958 + }
2959 + },
2960 + "operationId": "getVelociraptorAgentHealthcheck",
2961 + "tags": ["Healthcheck"]
2962 + }
2963 + },
2964 + "/healthcheck/agent/{agent_id}/velociraptor": {
2965 + "get": {
2966 + "summary": "Get Velociraptor agent healthcheck by agent ID",
2967 + "description": "Endpoint to get Velociraptor agent healthcheck by agent ID.",
2968 + "parameters": [
2969 + {
2970 + "name": "agent_id",
2971 + "in": "path",
2972 + "description": "The ID of the agent.",
2973 + "required": true,
2974 + "schema": {
2975 + "type": "string"
2976 + }
2977 + }
2978 + ],
2979 + "responses": {
2980 + "200": {
2981 + "description": "Successful operation",
2982 + "content": {
2983 + "application/json": {
2984 + "schema": {
2985 + "type": "object",
2986 + "properties": {
2987 + "agent_healthcheck": {
2988 + "type": "array",
2989 + "items": {
2990 + "type": "object",
2991 + "description": "Agent healthcheck details"
2992 + }
2993 + }
2994 + }
2995 + }
2996 + }
2997 + }
2998 + },
2999 + "default": {
3000 + "description": "Unexpected error",
3001 + "content": {
3002 + "application/json": {
3003 + "schema": {
3004 + "$ref": "#/components/schemas/Error"
3005 + }
3006 + }
3007 + }
3008 + }
3009 + },
3010 + "operationId": "getVelociraptorAgentHealthcheckByAgentID",
3011 + "tags": ["Healthcheck"]
3012 + }
3013 }
3014 },
3015 "components": {