added dnstwist integration (#93)
* added dnstwist integration * phishing dnstwist endpoint to swagger
taylor_socfortress committed
Sep 1, 2023 at 13:12 UTC
2c47270c4fd0a134a01b365fe43878fc251e280c
7 files changed
+300
backend/app/__init__.py
+2
@@ -53,6 +53,7 @@ from app.routes.alerts import bp as alerts_bp
53
from app.routes.connectors import bp as connectors_bp
54
from app.routes.customers import bp as customers_bp
55
from app.routes.dfir_iris import bp as dfir_iris_bp
56
+from app.routes.dnstwist import bp as dnstwist_bp
57
from app.routes.graylog import bp as graylog_bp
58
from app.routes.healthchecks import bp as healthchecks_bp
59
from app.routes.influxdb import bp as influxdb_bp
@@ -79,3 +80,4 @@ app.register_blueprint(smtp_bp) # Register the smtp blueprint
80
app.register_blueprint(healthchecks_bp) # Register the healthchecks blueprint
81
app.register_blueprint(threatintel_bp) # Register the threatintel blueprint
82
app.register_blueprint(customers_bp) # Register the customers blueprint
83
+app.register_blueprint(dnstwist_bp) # Register the dnstwist blueprint
backend/app/routes/dnstwist.py
new
+69
@@ -0,0 +1,69 @@
1
+from flask import Blueprint
2
+from flask import jsonify
3
+from flask import request
4
+from loguru import logger
5
+
6
+from app.services.dnstwist.analyze import DNSTwistService
7
+from app.services.dnstwist.universal import UniversalService
8
+
9
+bp = Blueprint("dnstwist", __name__)
10
+
11
+
12
+@bp.route("/dnstwist/registered", methods=["POST"])
13
+def receive_dnstwist_registered():
14
+ """
15
+ API Endpoint for receiving call to run dnstwist.
16
+ Accepts POST request with JSON body of `domain`.
17
+ """
18
+ logger.info(
19
+ "Received request to invoke DNStwist to detect similar registered domains.",
20
+ )
21
+ data = request.get_json()
22
+ logger.debug(f"Data: {data}")
23
+ if not data:
24
+ return jsonify({"message": "No data received."}), 400
25
+ if "domain" not in data:
26
+ return (
27
+ jsonify({"message": "Missing required data - domain.", "success": False}),
28
+ 400,
29
+ )
30
+
31
+ # Check if the domain is valid
32
+ if not UniversalService.is_domain(data["domain"]):
33
+ return jsonify({"message": "Invalid domain.", "success": False}), 400
34
+
35
+ # Invoke dnstwist service
36
+ dnstwist_service = DNSTwistService(data["domain"])
37
+ results = dnstwist_service.analyze_domain_registered()
38
+
39
+ # Return the results to the client
40
+ return jsonify(results), 200
41
+
42
+
43
+@bp.route("/dnstwist/phishing", methods=["POST"])
44
+def receive_dnstwist_phishing():
45
+ """
46
+ API Endpoint for receiving call to run dnstwist.
47
+ Accepts POST request with JSON body of `domain`.
48
+ """
49
+ logger.info("Received request to invoke DNStwist to detect potential Phishing.")
50
+ data = request.get_json()
51
+ logger.debug(f"Data: {data}")
52
+ if not data:
53
+ return jsonify({"message": "No data received."}), 400
54
+ if "domain" not in data:
55
+ return (
56
+ jsonify({"message": "Missing required data - domain.", "success": False}),
57
+ 400,
58
+ )
59
+
60
+ # Check if the domain is valid
61
+ if not UniversalService.is_domain(data["domain"]):
62
+ return jsonify({"message": "Invalid domain.", "success": False}), 400
63
+
64
+ # Invoke dnstwist service
65
+ dnstwist_service = DNSTwistService(data["domain"])
66
+ results = dnstwist_service.analyze_domain_phishing()
67
+
68
+ # Return the results to the client
69
+ return jsonify(results), 200
backend/app/services/dnstwist/__init__.py
backend/app/services/dnstwist/analyze.py
new
+68
@@ -0,0 +1,68 @@
1
+from typing import Any
2
+from typing import Optional
3
+
4
+import dnstwist
5
+from loguru import logger
6
+
7
+
8
+class DNSTwistService:
9
+ """
10
+ Service for handling operations related to dnstwist.
11
+
12
+ Attributes:
13
+ domain (Optional[str]): Domain to be analyzed by dnstwist.
14
+ """
15
+
16
+ def __init__(self, domain: Optional[str] = None):
17
+ """
18
+ Initialize a DNSTwistService instance.
19
+
20
+ Args:
21
+ domain (Optional[str]): Domain to be analyzed by dnstwist.
22
+ """
23
+ self.domain = domain
24
+
25
+ def set_domain(self, domain: str) -> None:
26
+ """
27
+ Set the domain to be analyzed by dnstwist.
28
+
29
+ Args:
30
+ domain (str): Domain to be analyzed.
31
+ """
32
+ self.domain = domain
33
+
34
+ def analyze_domain_registered(self) -> Any:
35
+ """
36
+ Analyze the domain using dnstwist and return the results for registered domains.
37
+
38
+ Returns:
39
+ Any: Results of the dnstwist analysis.
40
+ """
41
+ if self.domain is None:
42
+ raise ValueError("Domain must be set before analysis.")
43
+
44
+ logger.info("Analyzing domain for registered domains.")
45
+ data = dnstwist.run(domain=self.domain, registered=True, format="json")
46
+ logger.debug(f"Data: {data}")
47
+ return {"message": "Successfully analyzed domain.", "success": True, "data": data}
48
+
49
+ def analyze_domain_phishing(self) -> Any:
50
+ """
51
+ Analyze the domain using dnstwist and the lsh ssdeep which extracts the html from the domain's root site
52
+ and compares that to similar domains to detect cloned websites.
53
+
54
+ Returns:
55
+ Any: Results of the dnstwist analysis.
56
+ """
57
+ if self.domain is None:
58
+ raise ValueError("Domain must be set before analysis.")
59
+
60
+ logger.info("Analyzing domain for potential phishing.")
61
+ data = dnstwist.run(
62
+ domain=self.domain,
63
+ registered=True,
64
+ format="json",
65
+ lsh=True,
66
+ )
67
+ logger.debug(f"Data: {data}")
68
+ return {"message": "Successfully analyzed domain.", "success": True, "data": data}
backend/app/services/dnstwist/universal.py
new
+36
@@ -0,0 +1,36 @@
1
+import regex
2
+from loguru import logger
3
+
4
+
5
+class UniversalService:
6
+ """
7
+ Service for handling universal checks regardless of alert type.
8
+
9
+ Attributes:
10
+ _config_manager (Optional[ConfigManager]): ConfigManager object for accessing the config file.
11
+ _excluded_rule_ids (Optional[set[str]]): Set of rule ids that are excluded.
12
+ _valid_customer_codes (Optional[Dict[str, Tuple[int, str, str]]]):
13
+ Dictionary mapping customer codes to their corresponding data.
14
+ """
15
+
16
+ def __init__(self):
17
+ """
18
+ Initialize a UniversalService instance.
19
+ """
20
+
21
+ @staticmethod
22
+ def is_domain(domain: str) -> bool:
23
+ """
24
+ Check if the provided domain is valid.
25
+
26
+ Args:
27
+ domain (str): The domain to check.
28
+
29
+ Returns:
30
+ bool: True if the domain is valid, False otherwise.
31
+ """
32
+ logger.info(f"Checking if domain {domain} is valid.")
33
+ pattern = regex.compile(
34
+ r"^(?:[a-zA-Z0-9]+([-._]?[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,}$",
35
+ )
36
+ return bool(pattern.match(domain))
backend/app/static/swagger.json
+124
@@ -124,9 +124,133 @@
124
"description": "Find out more",
125
"url": "http://swagger.io"
126
}
127
+ },
128
+ {
129
+ "name": "DNSTwist",
130
+ "description": "DNSTwist Integration - https://github.com/elceef/dnstwist",
131
+ "externalDocs": {
132
+ "description": "Find out more",
133
+ "url": "http://swagger.io"
134
+ }
135
}
136
],
137
"paths": {
138
+ "/dnstwist/registered": {
139
+ "post": {
140
+ "tags": ["DNSTwist"],
141
+ "summary": "Get registered domains",
142
+ "description": "Endpoint to get registered domains from DNSTwist.",
143
+ "requestBody": {
144
+ "content": {
145
+ "application/json": {
146
+ "schema": {
147
+ "type": "object",
148
+ "properties": {
149
+ "domain": {
150
+ "type": "string",
151
+ "description": "Domain to check"
152
+ }
153
+ },
154
+ "required": ["domain"]
155
+ }
156
+ }
157
+ }
158
+ },
159
+ "responses": {
160
+ "200": {
161
+ "description": "Registered domains retrieved successfully.",
162
+ "content": {
163
+ "application/json": {
164
+ "schema": {
165
+ "type": "object",
166
+ "properties": {
167
+ "data": {
168
+ "type": "array",
169
+ "items": {
170
+ "type": "string"
171
+ }
172
+ }
173
+ }
174
+ }
175
+ }
176
+ }
177
+ },
178
+ "400": {
179
+ "description": "Invalid input",
180
+ "content": {
181
+ "application/json": {
182
+ "schema": {
183
+ "type": "object",
184
+ "properties": {
185
+ "error": {
186
+ "type": "string"
187
+ }
188
+ }
189
+ }
190
+ }
191
+ }
192
+ }
193
+ }
194
+ }
195
+ },
196
+ "/dnstwist/phishing": {
197
+ "post": {
198
+ "tags": ["DNSTwist"],
199
+ "summary": "Get phishing domains",
200
+ "description": "Endpoint to get phishing domains from DNSTwist.",
201
+ "requestBody": {
202
+ "content": {
203
+ "application/json": {
204
+ "schema": {
205
+ "type": "object",
206
+ "properties": {
207
+ "domain": {
208
+ "type": "string",
209
+ "description": "Domain to check"
210
+ }
211
+ },
212
+ "required": ["domain"]
213
+ }
214
+ }
215
+ }
216
+ },
217
+ "responses": {
218
+ "200": {
219
+ "description": "Phishing domains retrieved successfully.",
220
+ "content": {
221
+ "application/json": {
222
+ "schema": {
223
+ "type": "object",
224
+ "properties": {
225
+ "data": {
226
+ "type": "array",
227
+ "items": {
228
+ "type": "string"
229
+ }
230
+ }
231
+ }
232
+ }
233
+ }
234
+ }
235
+ },
236
+ "400": {
237
+ "description": "Invalid input",
238
+ "content": {
239
+ "application/json": {
240
+ "schema": {
241
+ "type": "object",
242
+ "properties": {
243
+ "error": {
244
+ "type": "string"
245
+ }
246
+ }
247
+ }
248
+ }
249
+ }
250
+ }
251
+ }
252
+ }
253
+ },
254
"/connectors": {
255
"get": {
256
"tags": ["Connectors"],
backend/requirements.in
+1
@@ -1,5 +1,6 @@
1
blueprint
2
dfir_iris_client
3
+dnstwist[full]
4
elasticsearch7==7.10.1
5
environs
6
flask