Cortex (#94)
* cortex connector * get cortex analyzers * run cortex analyzer and retrieve values
taylor_socfortress committed
Sep 1, 2023 at 16:17 UTC
146ac09ae8efd026061a82dc1adf45bf9a6dfe8b
9 files changed
+535
backend/app/__init__.py
+2
@@ -51,6 +51,7 @@ migrate = Migrate(app, db)
51
from app.routes.agents import bp as agents_bp
52
from app.routes.alerts import bp as alerts_bp
53
from app.routes.connectors import bp as connectors_bp
54
+from app.routes.cortex import bp as cortex_bp
55
from app.routes.customers import bp as customers_bp
56
from app.routes.dfir_iris import bp as dfir_iris_bp
57
from app.routes.dnstwist import bp as dnstwist_bp
@@ -81,3 +82,4 @@ app.register_blueprint(healthchecks_bp) # Register the healthchecks blueprint
82
app.register_blueprint(threatintel_bp) # Register the threatintel blueprint
83
app.register_blueprint(customers_bp) # Register the customers blueprint
84
app.register_blueprint(dnstwist_bp) # Register the dnstwist blueprint
85
+app.register_blueprint(cortex_bp) # Register the cortex blueprint
backend/app/models/connectors.py
+46
@@ -10,6 +10,7 @@ import grpc
10
import pika
11
import pyvelociraptor
12
import requests
13
+from cortex4py.api import Api
14
from elasticsearch7 import Elasticsearch
15
from flask import current_app
16
from loguru import logger
@@ -745,6 +746,50 @@ class RabbitMQConnector(Connector):
746
return {"connectionSuccessful": False, "response": None}
747
748
749
+class CortexConnector(Connector):
750
+ """
751
+ A connector for the Cortex service, a subclass of Connector.
752
+
753
+ Args:
754
+ connector_name (str): The name of the connector.
755
+ """
756
+
757
+ def __init__(self, connector_name: str):
758
+ super().__init__(attributes=self.get_connector_info_from_db(connector_name))
759
+
760
+ def verify_connection(self) -> Dict[str, Any]:
761
+ """
762
+ Verifies the connection to Cortex service.
763
+ """
764
+ logger.info(
765
+ f"Verifying the cortex connection to {self.attributes['connector_url']}",
766
+ )
767
+ try:
768
+ api = Api(self.attributes["connector_url"], self.attributes["connector_api_key"], verify_cert=False)
769
+
770
+ # Get Cortex Status
771
+ status = api.status
772
+ if status:
773
+ logger.info(
774
+ f"Connection to {self.attributes['connector_url']} successful",
775
+ )
776
+ # Update the connector_available table
777
+ self.update_connectors_available_table(self.attributes["connector_name"], True, True)
778
+ return {"connectionSuccessful": True}
779
+ else:
780
+ logger.error(f"Connection to {self.attributes['connector_url']} failed")
781
+ # Update the connector_available table
782
+ self.update_connectors_available_table(self.attributes["connector_name"], True, False)
783
+ return {"connectionSuccessful": False, "response": None}
784
+ except Exception as e:
785
+ logger.error(
786
+ f"Connection to {self.attributes['connector_url']} failed with error: {e}",
787
+ )
788
+ # Update the connector_available table
789
+ self.update_connectors_available_table(self.attributes["connector_name"], True, False)
790
+ return {"connectionSuccessful": False, "response": None}
791
+
792
+
793
class ConnectorFactory:
794
"""
795
This class represents a factory for creating connector instances.
@@ -802,3 +847,4 @@ connector_factory.register_creator("Sublime", "SublimeConnector")
847
connector_factory.register_creator("InfluxDB", "InfluxDBConnector")
848
connector_factory.register_creator("AskSocfortress", "AskSOCFortressConnector")
849
connector_factory.register_creator("SocfortressThreatIntel", "SocfortressThreatIntelConnector")
850
+connector_factory.register_creator("Cortex", "CortexConnector")
backend/app/models/models.py
+1
@@ -118,6 +118,7 @@ class Connectors(db.Model):
118
"influxdb": True,
119
"asksocfortress": True,
120
"socfortressthreatintel": True,
121
+ "cortex": True,
122
}
123
124
def __init__(
backend/app/routes/cortex.py
new
+63
@@ -0,0 +1,63 @@
1
+from flask import Blueprint
2
+from flask import jsonify
3
+from flask import request
4
+from loguru import logger
5
+
6
+from app.services.cortex.analyzers import AnalyzerService
7
+from app.services.cortex.universal import UniversalService
8
+
9
+bp = Blueprint("cortex", __name__)
10
+
11
+
12
+@bp.route("/cortex/analyzers", methods=["GET"])
13
+def receive_cortex_analyzers():
14
+ """
15
+ API Endpoint for receiving call to retrieve Cortex Analyzers.
16
+ Accepts GET request.
17
+ """
18
+ logger.info(
19
+ "Received request to invoke Cortex to list available analyzers.",
20
+ )
21
+
22
+ analyzers = AnalyzerService().get_analyzers()
23
+
24
+ return jsonify(analyzers), 200
25
+
26
+
27
+@bp.route("/cortex/analyzers/run", methods=["POST"])
28
+def receive_cortex_analyzers_run():
29
+ """
30
+ API Endpoint for receiving call to run Cortex Analyzers.
31
+ Accepts POST request with JSON body of `analyzer_name` and `artifact`.
32
+ """
33
+ logger.info(
34
+ "Received request to invoke Cortex to run an analyzer.",
35
+ )
36
+ data = request.get_json()
37
+ logger.debug(f"Data: {data}")
38
+ if not data:
39
+ return jsonify({"message": "No data received."}), 400
40
+ if "analyzer_name" not in data:
41
+ return (
42
+ jsonify({"message": "Missing required data - analyzer_name.", "success": False}),
43
+ 400,
44
+ )
45
+ if "artifact" not in data:
46
+ return (
47
+ jsonify({"message": "Missing required data - artifact.", "success": False}),
48
+ 400,
49
+ )
50
+
51
+ # Check if the artifact is valid
52
+ is_valid, data_type = UniversalService("Cortex").is_valid_datatype(value=data["artifact"])
53
+ if not is_valid:
54
+ return jsonify({"message": "Invalid artifact.", "success": False}), 400
55
+
56
+ # Run the analyzer
57
+ analyzer_service = AnalyzerService().run_and_wait_for_analyzer(
58
+ analyzer_name=data["analyzer_name"],
59
+ ioc_value=data["artifact"],
60
+ data_type=data_type,
61
+ )
62
+
63
+ return jsonify(analyzer_service), 200
backend/app/services/cortex/__init__.py
backend/app/services/cortex/analyzers.py
new
+168
@@ -0,0 +1,168 @@
1
+import time
2
+from typing import Any
3
+from typing import Dict
4
+from typing import List
5
+from typing import Optional
6
+from typing import Tuple
7
+from typing import Union
8
+
9
+from cortex4py.api import Api
10
+from loguru import logger
11
+
12
+from app.services.cortex.universal import UniversalService
13
+
14
+
15
+class AnalyzerService:
16
+ """
17
+ A service class for working with Cortex Analyzers.
18
+ Provides methods to fetch, process, and run analyzers from Cortex.
19
+ """
20
+
21
+ def __init__(self) -> None:
22
+ """
23
+ Initialize the AnalyzerService instance.
24
+ """
25
+ self.universal_service = UniversalService("Cortex")
26
+ self.connector_url, self.connector_api_key = self.initialize_connection_details()
27
+ self.api: Optional[Api] = self.create_api_instance()
28
+
29
+ def initialize_connection_details(self) -> Tuple[Optional[str], Optional[str]]:
30
+ """
31
+ Initialize the connection details for Cortex.
32
+
33
+ Returns:
34
+ tuple: Connector URL and API key
35
+ """
36
+ return self.universal_service.collect_cortex_details("Cortex")
37
+
38
+ def create_api_instance(self) -> Optional[Api]:
39
+ """
40
+ Create an API instance for Cortex.
41
+
42
+ Returns:
43
+ Optional[Api]: API instance if successful; None otherwise.
44
+ """
45
+ try:
46
+ return Api(self.connector_url, self.connector_api_key)
47
+ except Exception as e:
48
+ logger.error(f"Error initializing Cortex API: {e}")
49
+ return None
50
+
51
+ def get_analyzers(self) -> Dict[str, Union[bool, str, List[str]]]:
52
+ """
53
+ Retrieve analyzers from Cortex.
54
+
55
+ Returns:
56
+ Dict: Success status, a message, and fetched analyzers.
57
+ """
58
+ if self.api is None:
59
+ return {"success": False, "message": "API initialization failed"}
60
+
61
+ analyzers = self.fetch_analyzers()
62
+ return self.build_analyzer_response(analyzers)
63
+
64
+ def fetch_analyzers(self) -> List[Dict]:
65
+ """
66
+ Fetch analyzers from Cortex.
67
+
68
+ Returns:
69
+ List[Dict]: List of fetched analyzers.
70
+ """
71
+ return self.api.analyzers.find_all({}, range="all")
72
+
73
+ def build_analyzer_response(self, analyzers: List[Dict]) -> Dict[str, Union[bool, str, List[str]]]:
74
+ """
75
+ Build the response object for the fetched analyzers.
76
+
77
+ Args:
78
+ analyzers (List[Dict]): Fetched analyzers.
79
+
80
+ Returns:
81
+ Dict: Success status, message, and list of analyzer names.
82
+ """
83
+ try:
84
+ analyzer_names = [analyzer.name for analyzer in analyzers]
85
+ return {"success": True, "message": "Successfully fetched analyzers", "analyzers": analyzer_names}
86
+ except Exception as e:
87
+ logger.error(f"Error processing analyzers: {e}")
88
+ return {"success": False, "message": f"Error processing analyzers: {e}"}
89
+
90
+ def run_and_wait_for_analyzer(self, analyzer_name: str, ioc_value: str, data_type: str) -> Dict[str, Any]:
91
+ """
92
+ Initiates and monitors the execution of a specified Cortex analyzer.
93
+
94
+ Args:
95
+ analyzer_name (str): The name of the Cortex analyzer to run.
96
+ ioc_value (str): The Indicator of Compromise (IoC) to be analyzed.
97
+ data_type (str): The type of the IoC (e.g., "IP", "hash", "domain").
98
+
99
+ Returns:
100
+ Dict[str, Any]: A dictionary containing the success status, a message, and optionally the results of the analysis.
101
+ """
102
+ if self.api is None:
103
+ return {"success": False, "message": "API initialization failed"}
104
+ try:
105
+ job = self.api.analyzers.run_by_name(
106
+ analyzer_name,
107
+ {
108
+ "data": ioc_value,
109
+ "dataType": data_type,
110
+ "tlp": 1,
111
+ "message": "custom message sent to analyzer",
112
+ },
113
+ force=1,
114
+ )
115
+ return self.monitor_analyzer_job(job)
116
+ except Exception as e:
117
+ logger.error(f"Error running analyzer {analyzer_name}: {e}")
118
+ return {"success": False, "message": f"Error running analyzer {analyzer_name}: {e}"}
119
+
120
+ def monitor_analyzer_job(self, job: Any) -> Dict[str, Any]:
121
+ """
122
+ Monitors the progress of a running Cortex analyzer job.
123
+
124
+ Args:
125
+ job (Any): The job object representing the running analyzer.
126
+
127
+ Returns:
128
+ Dict[str, Any]: A dictionary containing the success status and a message.
129
+ """
130
+ r_json = job.json()
131
+ job_id = r_json["id"]
132
+ logger.info(f"Job ID is: {job_id}")
133
+
134
+ job_state = r_json["status"]
135
+ timer = 0
136
+
137
+ while job_state != "Success":
138
+ if timer == 60:
139
+ logger.error("Job failed to complete after 5 minutes.")
140
+ return {"success": False, "message": "Job timed out"}
141
+ timer += 1
142
+ logger.info(f"Timer is: {timer}")
143
+
144
+ if job_state == "Failure":
145
+ error_message = r_json["errorMessage"]
146
+ logger.error(f"Cortex Failure: {error_message}")
147
+ return {"success": False, "message": f"Analyzer failed: {error_message}"}
148
+
149
+ time.sleep(5)
150
+ followup_request = self.api.jobs.get_by_id(job_id)
151
+ r_json = followup_request.json()
152
+ job_state = r_json["status"]
153
+
154
+ return self.retrieve_final_report(job_id)
155
+
156
+ def retrieve_final_report(self, job_id: str) -> Dict[str, Any]:
157
+ """
158
+ Retrieves the final report of a completed Cortex analyzer job.
159
+
160
+ Args:
161
+ job_id (str): The ID of the completed job.
162
+
163
+ Returns:
164
+ Dict[str, Any]: A dictionary containing the success status, a message, and the report of the analysis.
165
+ """
166
+ report = self.api.jobs.get_report(job_id).report
167
+ final_report = report["full"]
168
+ return {"success": True, "message": "Analyzer ran successfully", "report": final_report}
backend/app/services/cortex/universal.py
new
+124
@@ -0,0 +1,124 @@
1
+import ipaddress
2
+from typing import Optional
3
+from typing import Tuple
4
+
5
+import regex
6
+from loguru import logger
7
+
8
+from app.models.connectors import Connector
9
+from app.models.connectors import connector_factory
10
+
11
+HASH_PATTERN = r"^[a-fA-F\d]{64}$"
12
+DOMAIN_PATTERN = r"^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$"
13
+HASH_REGEX = regex.compile(HASH_PATTERN, regex.IGNORECASE)
14
+DOMAIN_REGEX = regex.compile(DOMAIN_PATTERN, regex.IGNORECASE)
15
+
16
+
17
+class UniversalService:
18
+ """
19
+ A service class that encapsulates the logic for interfacing with Cortex. This class handles tasks like creating a session,
20
+ fetching and parsing data, and retrieving connector details.
21
+ """
22
+
23
+ def __init__(self, connector_name: str) -> None:
24
+ """
25
+ Initializes the UniversalService by collecting Cortex details associated with the specified connector name.
26
+
27
+ Args:
28
+ connector_name (str): The name of the Cortex connector.
29
+ """
30
+ self.connector_url, self.connector_api_key = self.collect_cortex_details(
31
+ connector_name,
32
+ )
33
+
34
+ def collect_cortex_details(
35
+ self,
36
+ connector_name: str,
37
+ ) -> Tuple[Optional[str], Optional[str]]:
38
+ """
39
+ Collects the details of the DFIR-IRIS connector.
40
+
41
+ Args:
42
+ connector_name (str): The name of the DFIR-IRIS connector.
43
+
44
+ Returns:
45
+ tuple: A tuple containing the connection URL and API key. If the connection is not successful, both elements of the tuple are
46
+ None.
47
+ """
48
+ connector_instance = connector_factory.create(connector_name, connector_name)
49
+ connection_successful = connector_instance.verify_connection()
50
+ if connection_successful:
51
+ connection_details = Connector.get_connector_info_from_db(connector_name)
52
+ return (
53
+ connection_details.get("connector_url"),
54
+ connection_details.get("connector_api_key"),
55
+ )
56
+ else:
57
+ return None, None
58
+
59
+ def is_valid_datatype(self, value: str) -> Tuple[bool, str]:
60
+ """
61
+ Check if input value is a valid data type - IPv4, hash, or domain.
62
+
63
+ Args:
64
+ value (str): The input value to check.
65
+
66
+ Returns:
67
+ tuple: A tuple containing a boolean indicating whether the input value is a valid data type and a string indicating the data type.
68
+ """
69
+ if self._is_valid_ipv4(value):
70
+ return True, "ip"
71
+ elif self._is_valid_hash(value):
72
+ return True, "hash"
73
+ elif self._is_valid_domain(value):
74
+ return True, "domain"
75
+ else:
76
+ return False, "Unknown"
77
+
78
+ def _is_valid_ipv4(self, value: str) -> bool:
79
+ """
80
+ Check if input value is a valid IPv4 address.
81
+
82
+ Args:
83
+ value (str): The input value to check.
84
+
85
+ Returns:
86
+ bool: True if the input value is a valid IPv4 address, False otherwise.
87
+ """
88
+ try:
89
+ ipaddress.IPv4Address(value)
90
+ return True
91
+ except ValueError:
92
+ return False
93
+
94
+ def _is_valid_hash(self, value: str) -> bool:
95
+ """
96
+ Check if input value is a valid hash.
97
+
98
+ Args:
99
+ value (str): The input value to check.
100
+
101
+ Returns:
102
+ bool: True if the input value is a valid hash, False otherwise.
103
+ """
104
+ try:
105
+ return bool(HASH_REGEX.match(value))
106
+ except Exception as e:
107
+ logger.error(f"Error validating hash: {e}")
108
+ return False
109
+
110
+ def _is_valid_domain(self, value: str) -> bool:
111
+ """
112
+ Check if input value is a valid domain.
113
+
114
+ Args:
115
+ value (str): The input value to check.
116
+
117
+ Returns:
118
+ bool: True if the input value is a valid domain, False otherwise.
119
+ """
120
+ try:
121
+ return bool(DOMAIN_REGEX.match(value))
122
+ except Exception as e:
123
+ logger.error(f"Error validating domain: {e}")
124
+ return False
backend/app/static/swagger.json
+128
@@ -132,9 +132,137 @@
132
"description": "Find out more",
133
"url": "http://swagger.io"
134
}
135
+ },
136
+ {
137
+ "name": "Cortex",
138
+ "description": "Everything about Cortex",
139
+ "externalDocs": {
140
+ "description": "Find out more",
141
+ "url": "http://swagger.io"
142
+ }
143
}
144
],
145
"paths": {
146
+ "/cortex/analyzers": {
147
+ "get": {
148
+ "tags": ["Cortex"],
149
+ "summary": "List all available analyzers",
150
+ "description": "Endpoint to list all available analyzers. It processes each analyzer to verify the connection and returns the results.",
151
+ "responses": {
152
+ "200": {
153
+ "description": "A JSON response containing the list of all available analyzers along with their connection verification status.",
154
+ "content": {
155
+ "application/json": {
156
+ "schema": {
157
+ "type": "array",
158
+ "items": {
159
+ "type": "object",
160
+ "properties": {
161
+ "connectionSuccessful": {
162
+ "type": "boolean"
163
+ },
164
+ "analyzer_api_key": {
165
+ "type": "string",
166
+ "nullable": true
167
+ },
168
+ "analyzer_last_updated": {
169
+ "type": "string",
170
+ "format": "date-time"
171
+ },
172
+ "analyzer_name": {
173
+ "type": "string"
174
+ },
175
+ "analyzer_password": {
176
+ "type": "string",
177
+ "nullable": true
178
+ },
179
+ "analyzer_type": {
180
+ "type": "string"
181
+ },
182
+ "analyzer_url": {
183
+ "type": "string"
184
+ },
185
+ "analyzer_username": {
186
+ "type": "string",
187
+ "nullable": true
188
+ },
189
+ "id": {
190
+ "type": "integer"
191
+ },
192
+ "name": {
193
+ "type": "string"
194
+ }
195
+ }
196
+ }
197
+ }
198
+ }
199
+ }
200
+ }
201
+ }
202
+ }
203
+ },
204
+ "/cortex/analyzers/run": {
205
+ "post": {
206
+ "tags": ["Cortex"],
207
+ "summary": "Run Cortex analyzer",
208
+ "description": "Endpoint to run Cortex analyzer.",
209
+ "requestBody": {
210
+ "content": {
211
+ "application/json": {
212
+ "schema": {
213
+ "type": "object",
214
+ "properties": {
215
+ "analyzer_name": {
216
+ "type": "string",
217
+ "description": "Analyzer name"
218
+ },
219
+ "artifact": {
220
+ "type": "string",
221
+ "description": "Artifact"
222
+ }
223
+ },
224
+ "required": ["analyzer_name", "artifact"]
225
+ }
226
+ }
227
+ }
228
+ },
229
+ "responses": {
230
+ "200": {
231
+ "description": "Cortex analyzer run successfully.",
232
+ "content": {
233
+ "application/json": {
234
+ "schema": {
235
+ "type": "object",
236
+ "properties": {
237
+ "data": {
238
+ "type": "array",
239
+ "items": {
240
+ "type": "string"
241
+ }
242
+ }
243
+ }
244
+ }
245
+ }
246
+ }
247
+ },
248
+ "400": {
249
+ "description": "Invalid input",
250
+ "content": {
251
+ "application/json": {
252
+ "schema": {
253
+ "type": "object",
254
+ "properties": {
255
+ "error": {
256
+ "type": "string"
257
+ }
258
+ }
259
+ }
260
+ }
261
+ }
262
+ }
263
+ }
264
+ }
265
+ },
266
"/dnstwist/registered": {
267
"post": {
268
"tags": ["DNSTwist"],
backend/requirements.in
+3
@@ -1,4 +1,5 @@
1
blueprint
2
+cortex4py
3
dfir_iris_client
4
dnstwist[full]
5
elasticsearch7==7.10.1
@@ -10,6 +11,7 @@ flask-migrate
11
flask-sqlalchemy
12
flask-swagger-ui
13
flask_cors
14
+libmagic
15
loguru
16
marshmallow-sqlalchemy
17
matplotlib
@@ -18,6 +20,7 @@ openai
20
pika
21
psycopg2-binary
22
pytest
23
+python-magic
24
pyvelociraptor~=0.1
25
regex
26
reportlab