main
py 551 lines 21.7 KB
Raw
1 import json
2 import os
3 from datetime import datetime
4
5 import aiofiles
6 from fastapi import HTTPException
7 from loguru import logger
8 from sqlalchemy import and_
9 from sqlalchemy import update
10 from sqlalchemy.ext.asyncio import AsyncSession
11
12 from app.connectors.grafana.schema.dashboards import CrowdstrikeDashboard
13 from app.connectors.grafana.schema.dashboards import DashboardProvisionRequest
14 from app.connectors.grafana.services.dashboards import provision_dashboards
15 from app.connectors.grafana.utils.universal import create_grafana_client
16 from app.connectors.graylog.services.collector import (
17 get_content_pack_id_by_content_pack_name,
18 )
19 from app.connectors.graylog.services.collector import get_input_id_by_input_name
20 from app.connectors.graylog.services.collector import get_stream_id_by_stream_name
21 from app.connectors.graylog.services.streams import assign_stream_to_index
22 from app.connectors.graylog.utils.universal import send_post_request
23 from app.connectors.wazuh_indexer.services.monitoring import (
24 output_shard_number_to_be_set_based_on_nodes,
25 )
26 from app.customer_provisioning.schema.grafana import GrafanaDatasource
27 from app.customer_provisioning.schema.grafana import GrafanaDataSourceCreationResponse
28 from app.customer_provisioning.schema.graylog import GraylogIndexSetCreationResponse
29 from app.customer_provisioning.schema.graylog import StreamConnectionToPipelineRequest
30 from app.customer_provisioning.schema.graylog import TimeBasedIndexSet
31 from app.customer_provisioning.schema.provision import ProvisionNewCustomer
32 from app.customer_provisioning.services.grafana import create_grafana_folder
33 from app.customer_provisioning.services.grafana import get_opensearch_version
34 from app.customer_provisioning.services.graylog import connect_stream_to_pipeline
35 from app.customer_provisioning.services.graylog import get_pipeline_id
36 from app.customers.routes.customers import get_customer_meta
37 from app.integrations.crowdstrike.schema.provision import CrowdstrikeCustomerDetails
38 from app.integrations.crowdstrike.schema.provision import ProvisionCrowdstrikeAuthKeys
39 from app.integrations.crowdstrike.schema.provision import ProvisionCrowdstrikeResponse
40 from app.integrations.models.customer_integration_settings import CustomerIntegrations
41 from app.network_connectors.models.network_connectors import (
42 CustomerNetworkConnectorsMeta,
43 )
44 from app.stack_provisioning.graylog.schema.provision import ContentPackKeywords
45 from app.stack_provisioning.graylog.schema.provision import (
46 ProvisionNetworkContentPackRequest,
47 )
48 from app.stack_provisioning.graylog.services.provision import (
49 provision_content_pack_network_connector,
50 )
51 from app.utils import get_connector_attribute
52 from app.utils import get_customer_meta_attribute
53
54
55 #### ! GRAYLOG ! ####
56 async def build_index_set_config(request: CrowdstrikeCustomerDetails) -> TimeBasedIndexSet:
57 """
58 Build the configuration for a time-based index set.
59
60 Args:
61 request (CrowdstrikeCustomerDetails): The request object containing customer information.
62
63 Returns:
64 TimeBasedIndexSet: The configured time-based index set.
65 """
66 return TimeBasedIndexSet(
67 title=f"{request.customer_name} - CROWDSTRIKE EVENTS",
68 description=f"{request.customer_name} - CROWDSTRIKE EVENTS",
69 index_prefix=f"crowdstrike-{request.customer_code}",
70 rotation_strategy_class="org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy",
71 rotation_strategy={
72 "type": "org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig",
73 "rotation_period": "P1D",
74 "rotate_empty_index_set": False,
75 "max_rotation_period": None,
76 },
77 retention_strategy_class="org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy",
78 retention_strategy={
79 "type": "org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig",
80 "max_number_of_indices": request.hot_data_retention,
81 },
82 creation_date=datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
83 index_analyzer="standard",
84 shards=await output_shard_number_to_be_set_based_on_nodes(),
85 replicas=request.index_replicas,
86 index_optimization_max_num_segments=1,
87 index_optimization_disabled=False,
88 writable=True,
89 field_type_refresh_interval=5000,
90 )
91
92
93 # Function to send the POST request and handle the response
94 async def send_index_set_creation_request(
95 index_set: TimeBasedIndexSet,
96 ) -> GraylogIndexSetCreationResponse:
97 """
98 Sends a request to create an index set in Graylog.
99
100 Args:
101 index_set (TimeBasedIndexSet): The index set to be created.
102
103 Returns:
104 GraylogIndexSetCreationResponse: The response from Graylog after creating the index set.
105 """
106 json_index_set = json.dumps(index_set.model_dump())
107 logger.info(f"json_index_set set: {json_index_set}")
108 response_json = await send_post_request(
109 endpoint="/api/system/indices/index_sets",
110 data=index_set.model_dump(),
111 )
112 return GraylogIndexSetCreationResponse(**response_json)
113
114
115 # Refactored create_index_set function
116 async def create_index_set(
117 request: ProvisionNewCustomer,
118 ) -> GraylogIndexSetCreationResponse:
119 """
120 Creates an index set for a new customer.
121
122 Args:
123 request (ProvisionNewCustomer): The request object containing the customer information.
124
125 Returns:
126 GraylogIndexSetCreationResponse: The response object containing the result of the index set creation.
127 """
128 logger.info(f"Creating index set for customer {request.customer_name}")
129 index_set_config = await build_index_set_config(request)
130 return await send_index_set_creation_request(index_set_config)
131
132
133 async def provision_content_pack(customer_details):
134 """
135 Provisions a content pack for a customer.
136
137 Args:
138 customer_details (CustomerDetails): The details of the customer.
139
140 Returns:
141 ContentPack: The provisioned content pack.
142 """
143 return await provision_content_pack_network_connector(
144 content_pack_request=ProvisionNetworkContentPackRequest(
145 content_pack_name="CROWDSTRIKE",
146 keywords=ContentPackKeywords(
147 customer_name=customer_details.customer_name,
148 customer_code=customer_details.customer_code,
149 protocol_type=customer_details.protocal_type,
150 syslog_port=customer_details.syslog_port,
151 ),
152 ),
153 )
154
155
156 async def get_stream_and_index_ids(customer_details):
157 """
158 Retrieves the stream ID and index ID for a given customer.
159
160 Args:
161 customer_details (CustomerDetails): The details of the customer.
162
163 Returns:
164 tuple: A tuple containing the stream ID and index ID.
165 """
166 stream_id = await get_stream_id_by_stream_name(stream_name=f"{customer_details.customer_name} - CROWDSTRIKE LOGS AND EVENTS")
167 index_id = (await create_index_set(request=customer_details)).data.id
168 content_pack_stream_id = await get_content_pack_id_by_content_pack_name(
169 content_pack_name=f"{customer_details.customer_name}_CROWDSTRIKE_STREAM",
170 )
171 if customer_details.protocal_type == "TCP":
172 content_pack_input_id = await get_content_pack_id_by_content_pack_name(
173 content_pack_name=f"{customer_details.customer_name}_CROWDSTRIKE_INPUT_TCP",
174 )
175 elif customer_details.protocal_type == "UDP":
176 content_pack_input_id = await get_content_pack_id_by_content_pack_name(
177 content_pack_name=f"{customer_details.customer_name}_CROWDSTRIKE_INPUT_SYSLOG_UDP",
178 )
179 return stream_id, index_id, content_pack_stream_id, content_pack_input_id
180
181
182 #### ! GRAFANA ! ####
183 async def create_grafana_datasource(
184 customer_code: str,
185 session: AsyncSession,
186 ) -> GrafanaDataSourceCreationResponse:
187 """
188 Creates a Grafana datasource for the specified customer.
189
190 Args:
191 customer_code (str): The customer code.
192 session (AsyncSession): The async session.
193
194 Returns:
195 GrafanaDataSourceCreationResponse: The response containing the created datasource details.
196 """
197 logger.info("Creating Grafana datasource")
198 grafana_client = await create_grafana_client("Grafana")
199 # Switch to the newly created organization
200 grafana_client.user.switch_actual_user_organisation(
201 (await get_customer_meta(customer_code, session)).customer_meta.customer_meta_grafana_org_id,
202 )
203 datasource_payload = GrafanaDatasource(
204 name="CROWDSTRIKE",
205 type="grafana-opensearch-datasource",
206 typeName="OpenSearch",
207 access="proxy",
208 url=await get_connector_attribute(
209 connector_id=1,
210 column_name="connector_url",
211 session=session,
212 ),
213 database=f"crowdstrike-{customer_code}*",
214 basicAuth=True,
215 basicAuthUser=await get_connector_attribute(
216 connector_id=1,
217 column_name="connector_username",
218 session=session,
219 ),
220 secureJsonData={
221 "basicAuthPassword": await get_connector_attribute(
222 connector_id=1,
223 column_name="connector_password",
224 session=session,
225 ),
226 },
227 isDefault=False,
228 jsonData={
229 "database": f"crowdstrike-{customer_code}*",
230 "flavor": "opensearch",
231 "includeFrozen": False,
232 "logLevelField": "severity",
233 "logMessageField": "summary",
234 "maxConcurrentShardRequests": 5,
235 "pplEnabled": True,
236 "timeField": "timestamp",
237 "tlsSkipVerify": True,
238 "version": await get_opensearch_version(),
239 },
240 readOnly=True,
241 )
242 results = grafana_client.datasource.create_datasource(
243 datasource=datasource_payload.model_dump(),
244 )
245 return GrafanaDataSourceCreationResponse(**results)
246
247
248 async def create_customer_network_connector_meta(
249 customer_details,
250 stream_id,
251 index_id,
252 content_pack_stream_id,
253 content_pack_input_id,
254 session,
255 ):
256 """
257 Create a CustomerNetworkConnectorsMeta object with the provided details.
258
259 Args:
260 customer_details (CustomerDetails): Details of the customer.
261 stream_id (int): ID of the Graylog stream.
262 index_id (int): ID of the Graylog index.
263 session (Session): Database session.
264
265 Returns:
266 CustomerNetworkConnectorsMeta: The created CustomerNetworkConnectorsMeta object.
267 """
268 return CustomerNetworkConnectorsMeta(
269 customer_code=customer_details.customer_code,
270 network_connector_name="CROWDSTRIKE",
271 graylog_stream_id=stream_id,
272 graylog_input_id=(await get_input_id_by_input_name(input_name=f"{customer_details.customer_name} - CROWDSTRIKE LOGS AND EVENTS")),
273 graylog_pipeline_id=((await get_pipeline_id(subscription="CROWDSTRIKE"))[0]),
274 graylog_content_pack_input_id=content_pack_input_id,
275 graylog_content_pack_stream_id=content_pack_stream_id,
276 grafana_org_id=(
277 await get_customer_meta_attribute(
278 session=session,
279 customer_code=customer_details.customer_code,
280 column_name="customer_meta_grafana_org_id",
281 )
282 ),
283 graylog_index_id=index_id,
284 grafana_dashboard_folder_id=None,
285 grafana_datasource_uid=None,
286 )
287
288
289 async def validate_grafana_organization_id(customer_code, session):
290 """
291 Validate the Grafana organization ID for the customer.
292
293 Args:
294 customer_code (str): The customer code.
295 session (Session): Database session.
296
297 Returns:
298 int: The Grafana organization ID.
299 """
300 return await get_customer_meta_attribute(session=session, customer_code=customer_code, column_name="customer_meta_grafana_org_id")
301
302
303 async def provision_crowdstrike(
304 customer_details: CrowdstrikeCustomerDetails,
305 keys: ProvisionCrowdstrikeAuthKeys,
306 session: AsyncSession,
307 ) -> ProvisionCrowdstrikeResponse:
308 """
309 Provisions a Crowdstrike customer by performing the following steps:
310 1. Provisions the content pack for the customer.
311 2. Retrieves the stream and index IDs for the customer.
312 3. Creates customer network connector metadata.
313 4. Assigns the stream to the index.
314 5. Retrieves the pipeline ID for the "CROWDSTRIKE" subscription.
315 6. Connects the stream to the pipeline.
316 7. Inserts the customer network connector metadata into the database.
317 8. Creates a directory for the customer to store the docker compose and falconhose cfg.
318
319 Args:
320 customer_details (CrowdstrikeCustomerDetails): The details of the Crowdstrike customer.
321 keys (ProvisionCrowdstrikeKeys): The keys required for provisioning.
322 session (AsyncSession): The database session.
323
324 Returns:
325 None
326 """
327 # If customer name contains a space, replace it with a _
328 if " " in customer_details.customer_name:
329 customer_details.customer_name = customer_details.customer_name.replace(" ", "_")
330 if await validate_grafana_organization_id(customer_details.customer_code, session) is None:
331 raise HTTPException(status_code=404, detail="Grafana organization ID not found. Please provision Grafana for the customer first.")
332 await provision_content_pack(customer_details)
333 stream_id, index_id, content_pack_stream_id, content_pack_input_id = await get_stream_and_index_ids(customer_details)
334 customer_network_connector_meta = await create_customer_network_connector_meta(
335 customer_details,
336 stream_id,
337 index_id,
338 content_pack_stream_id,
339 content_pack_input_id,
340 session,
341 )
342 await assign_stream_to_index(stream_id=stream_id, index_id=index_id)
343 pipeline_id = await get_pipeline_id(subscription="CROWDSTRIKE")
344 await connect_stream_to_pipeline(stream_and_pipeline=StreamConnectionToPipelineRequest(stream_id=stream_id, pipeline_ids=pipeline_id))
345 # Grafana Deployment
346 customer_network_connector_meta.grafana_datasource_uid = (
347 await create_grafana_datasource(
348 customer_code=customer_details.customer_code,
349 session=session,
350 )
351 ).datasource.uid
352 grafana_folder = await create_grafana_folder(
353 organization_id=(
354 await get_customer_meta(
355 customer_details.customer_code,
356 session,
357 )
358 ).customer_meta.customer_meta_grafana_org_id,
359 folder_title="CROWDSTRIKE",
360 )
361 await provision_dashboards(
362 DashboardProvisionRequest(
363 dashboards=[dashboard.name for dashboard in CrowdstrikeDashboard],
364 organizationId=(
365 await get_customer_meta(
366 customer_details.customer_code,
367 session,
368 )
369 ).customer_meta.customer_meta_grafana_org_id,
370 folderId=grafana_folder.id,
371 datasourceUid=customer_network_connector_meta.grafana_datasource_uid,
372 ),
373 )
374 customer_network_connector_meta.grafana_dashboard_folder_id = grafana_folder.uid
375 await insert_into_customer_network_connectors_meta_table(
376 customer_network_connectors_meta=customer_network_connector_meta,
377 session=session,
378 )
379 await create_customer_directory_if_needed(customer_name=customer_details.customer_name)
380 file = await load_and_replace_docker_compose(customer_name=customer_details.customer_name)
381 await save_uploaded_file(
382 file=file,
383 filename=f"{customer_details.customer_name}_docker-compose.yml",
384 customer_name=customer_details.customer_name,
385 )
386 await load_and_replace_falconhose_cfg(customer_details=customer_details, keys=keys, session=session)
387
388 await update_customer_integration_table(
389 customer_code=customer_details.customer_code,
390 session=session,
391 )
392
393 return ProvisionCrowdstrikeResponse(
394 message="Crowdstrike customer provisioned successfully",
395 success=True,
396 )
397
398
399 async def insert_into_customer_network_connectors_meta_table(
400 customer_network_connectors_meta: CustomerNetworkConnectorsMeta,
401 session: AsyncSession,
402 ) -> None:
403 """
404 Insert the customer network connectors meta into the database.
405
406 Args:
407 customer_network_connectors_meta (CustomerNetworkConnectorsMeta): The customer network connectors meta to insert.
408 session (AsyncSession): The async session object for database operations.
409
410 Returns:
411 None
412 """
413 logger.info("Inserting customer network connectors meta into the database")
414 session.add(customer_network_connectors_meta)
415 await session.commit()
416 logger.info("Customer network connectors meta inserted successfully")
417 return None
418
419
420 # ! Add the docker-compose.yml file to the `data` folder
421 project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
422 UPLOAD_FOLDER = os.path.join(project_root, "data")
423
424
425 async def create_customer_directory_if_needed(customer_name: str):
426 """
427 Create a directory for the customer in the UPLOAD_FOLDER if it doesn't exist.
428
429 Args:
430 customer_name (str): The name of the customer.
431 """
432 # Create the path to the customer's directory
433 # If customer name contains a space, replace it with a _
434 if " " in customer_name:
435 customer_name = customer_name.replace(" ", "_")
436 customer_directory = os.path.join(UPLOAD_FOLDER, customer_name)
437 # Check if the directory exists
438 if not os.path.exists(customer_directory):
439 # If it doesn't exist, create it
440 os.makedirs(customer_directory)
441
442
443 async def load_and_replace_docker_compose(customer_name: str):
444 """
445 Load the docker-compose.yml file and replace the placeholder with the customer name.
446
447 Args:
448 customer_name (str): The name of the customer.
449
450 Returns:
451 str: The content of the docker-compose.yml file with the placeholder replaced.
452 """
453 # Get the current directory:
454 current_directory = os.path.dirname(os.path.abspath(__file__))
455 # Go up one level
456 parent_directory = os.path.dirname(current_directory)
457 # If customer name contains a space, replace it with a _
458 if " " in customer_name:
459 customer_name = customer_name.replace(" ", "_")
460 # Open the docker-compose.yml file and read the content
461 with open(os.path.join(parent_directory, "templates", "docker-compose.yml"), "r") as file:
462 data = file.read()
463 data = data.replace("CUSTOMER_NAME", customer_name)
464 return data
465
466
467 async def save_uploaded_file(file, filename, customer_name):
468 """
469 Save the uploaded file to the server.
470
471 Args:
472 file: The file to save.
473 filename: The name of the file.
474
475 Returns:
476 str: The path to the saved file.
477 """
478 # If customer name contains a space, replace it with a _
479 if " " in customer_name:
480 customer_name = customer_name.replace(" ", "_")
481 customer_upload_folder = os.path.join(UPLOAD_FOLDER, customer_name)
482 async with aiofiles.open(os.path.join(customer_upload_folder, filename), "wb") as f:
483 await f.write(file.encode())
484 return os.path.join(customer_upload_folder, filename)
485
486
487 async def load_and_replace_falconhose_cfg(
488 customer_details: CrowdstrikeCustomerDetails,
489 keys: ProvisionCrowdstrikeAuthKeys,
490 session: AsyncSession,
491 ):
492 """
493 Load the falconhose.cfg file and replace the placeholders with the customer details.
494
495 Args:
496 customer_details (CrowdstrikeCustomerDetails): The details of the customer.
497 keys (ProvisionCrowdstrikeAuthKeys): The authentication keys for Crowdstrike.
498
499 Returns:
500 str: The content of the falconhose.cfg file with the placeholders replaced.
501 """
502 # Get the current directory:
503 current_directory = os.path.dirname(os.path.abspath(__file__))
504 # Go up one level
505 parent_directory = os.path.dirname(current_directory)
506 connector_url = str(await get_connector_attribute(connector_id=3, column_name="connector_url", session=session))
507 connector_url = connector_url.replace("https://", "").replace("http://", "").replace(":9000", "")
508 # Open the falconhose.cfg file and read the content
509 with open(os.path.join(parent_directory, "templates", "cs.falconhoseclient.cfg"), "r") as file:
510 data = file.read()
511 data = data.replace("REPLACE_BASE_URL", keys.BASE_URL)
512 data = data.replace("REPLACE_CLIENT_ID", keys.CLIENT_ID)
513 data = data.replace("REPLACE_CLIENT_SECRET", keys.CLIENT_SECRET)
514 data = data.replace("REPLACE_SYSLOG_HOST", connector_url)
515 data = data.replace("REPLACE_SYSLOG_PORT", keys.SYSLOG_PORT)
516 # Save the file
517 # If customer name contains a space, replace it with a _
518 if " " in customer_details.customer_name:
519 customer_details.customer_name = customer_details.customer_name.replace(" ", "_")
520 customer_upload_folder = os.path.join(UPLOAD_FOLDER, customer_details.customer_name)
521 async with aiofiles.open(os.path.join(customer_upload_folder, "cs.falconhoseclient.cfg"), "w") as f:
522 await f.write(data)
523 return os.path.join(customer_upload_folder, "cs.falconhoseclient.cfg")
524
525
526 async def update_customer_integration_table(
527 customer_code: str,
528 session: AsyncSession,
529 ) -> None:
530 """
531 Updates the `customer_integrations` table to set the `deployed` column to True where the `customer_code`
532 matches the given customer code and the `integration_service_name` is "Crowdstrike".
533
534 Args:
535 customer_code (str): The customer code.
536 session (AsyncSession): The async session object for making HTTP requests.
537 """
538 logger.info(f"Updating customer integrations table for customer {customer_code}")
539 await session.execute(
540 update(CustomerIntegrations)
541 .where(
542 and_(
543 CustomerIntegrations.customer_code == customer_code,
544 CustomerIntegrations.integration_service_name == "Crowdstrike",
545 ),
546 )
547 .values(deployed=True),
548 )
549 await session.commit()
550
551 return None