main
py 1,080 lines 43 KB
Raw
1 from datetime import datetime
2 from typing import Dict
3 from typing import List
4 from typing import Optional
5 from typing import Union
6
7 import httpx
8 from fastapi import HTTPException
9 from loguru import logger
10 from sqlalchemy.ext.asyncio import AsyncSession
11
12 from app.connectors.velociraptor.schema.artifacts import ArtifactParametersResponse
13 from app.connectors.velociraptor.schema.artifacts import ArtifactReccomendationRequest
14 from app.connectors.velociraptor.schema.artifacts import ArtifactReccomendationResponse
15 from app.connectors.velociraptor.schema.artifacts import Artifacts
16 from app.connectors.velociraptor.schema.artifacts import ArtifactsResponse
17 from app.connectors.velociraptor.schema.artifacts import CollectArtifactBody
18 from app.connectors.velociraptor.schema.artifacts import CollectArtifactResponse
19 from app.connectors.velociraptor.schema.artifacts import CollectFileBody
20 from app.connectors.velociraptor.schema.artifacts import ParameterKeyValue
21 from app.connectors.velociraptor.schema.artifacts import QuarantineBody
22 from app.connectors.velociraptor.schema.artifacts import QuarantineResponse
23 from app.connectors.velociraptor.schema.artifacts import RunCommandBody
24 from app.connectors.velociraptor.schema.artifacts import RunCommandResponse
25 from app.connectors.velociraptor.utils.universal import UniversalService
26
27
28 def create_query(query: str) -> str:
29 """
30 Create a query string.
31
32 Args:
33 query (str): The query to be executed.
34
35 Returns:
36 str: The created query string.
37 """
38 return query
39
40
41 def get_artifact_key(analyzer_body: CollectArtifactBody) -> str:
42 """
43 Construct the artifact key.
44
45 Args:
46 analyzer_body: The collector body with artifact details
47
48 Returns:
49 str: The constructed artifact key.
50 """
51 action = getattr(analyzer_body, "action", None)
52 command = getattr(analyzer_body, "command", None)
53 parameters = getattr(analyzer_body, "parameters", None)
54
55 if action == "quarantine":
56 return (
57 f'collect_client(org_id="{analyzer_body.velociraptor_org}", client_id="{analyzer_body.velociraptor_id}", '
58 f'artifacts=["{analyzer_body.artifact_name}"], '
59 f"spec=dict(`{analyzer_body.artifact_name}`=dict()))"
60 )
61 elif action == "remove_quarantine":
62 return (
63 f'collect_client(org_id="{analyzer_body.velociraptor_org}", client_id="{analyzer_body.velociraptor_id}", '
64 f'artifacts=["{analyzer_body.artifact_name}"], '
65 f'spec=dict(`{analyzer_body.artifact_name}`=dict(`RemovePolicy`="Y")))'
66 )
67 elif command is not None:
68 return (
69 f"collect_client(org_id='{analyzer_body.velociraptor_org}', client_id='{analyzer_body.velociraptor_id}', "
70 f"urgent=true, artifacts=['{analyzer_body.artifact_name}'], "
71 f"env=dict(Command='{analyzer_body.command}'))"
72 )
73 elif parameters is not None:
74 # Parameters are provided, will be included in the query
75 return (
76 f"collect_client(org_id='{analyzer_body.velociraptor_org}', client_id='{analyzer_body.velociraptor_id}', "
77 f"artifacts=['{analyzer_body.artifact_name}'])"
78 )
79 else:
80 return (
81 f"collect_client(org_id='{analyzer_body.velociraptor_org}', client_id='{analyzer_body.velociraptor_id}', "
82 f"artifacts=['{analyzer_body.artifact_name}'])"
83 )
84
85
86 async def get_artifacts() -> ArtifactsResponse:
87 """
88 Get all artifacts from Velociraptor.
89
90 Returns:
91 ArtifactsResponse: A dictionary containing the artifacts.
92 """
93 logger.info("Fetching artifacts from Velociraptor")
94 velociraptor_service = await UniversalService.create("Velociraptor")
95 query = create_query("SELECT name,description,parameters FROM artifact_definitions()")
96 all_artifacts = velociraptor_service.execute_query(query)
97 try:
98 if all_artifacts["success"]:
99 artifacts = [Artifacts(**artifact) for artifact in all_artifacts["results"]]
100 return ArtifactsResponse(
101 success=True,
102 message="All artifacts retrieved",
103 artifacts=artifacts,
104 )
105 else:
106 raise HTTPException(
107 status_code=500,
108 detail=f"Failed to get all artifacts: {all_artifacts['message']}",
109 )
110 except Exception as err:
111 logger.error(f"Failed to get all artifacts: {err}")
112 raise HTTPException(
113 status_code=500,
114 detail=f"Failed to get all artifacts: {err}",
115 )
116
117
118 async def get_artifact_by_name(artifact_name: str) -> ArtifactsResponse:
119 """
120 Get a specific artifact by name from Velociraptor.
121
122 Args:
123 artifact_name (str): The name of the artifact to retrieve.
124
125 Returns:
126 ArtifactsResponse: A response containing the specific artifact.
127 """
128 logger.info(f"Fetching artifact '{artifact_name}' from Velociraptor")
129 velociraptor_service = await UniversalService.create("Velociraptor")
130
131 # Query for a specific artifact by name
132 query = create_query(f"SELECT name,description,parameters FROM artifact_definitions() WHERE name = '{artifact_name}'")
133 artifact_result = velociraptor_service.execute_query(query)
134
135 try:
136 if artifact_result["success"]:
137 if artifact_result["results"]:
138 artifacts = [Artifacts(**artifact) for artifact in artifact_result["results"]]
139 return ArtifactsResponse(
140 success=True,
141 message=f"Artifact '{artifact_name}' retrieved successfully",
142 artifacts=artifacts,
143 )
144 else:
145 return ArtifactsResponse(
146 success=True,
147 message=f"Artifact '{artifact_name}' not found",
148 artifacts=[],
149 )
150 else:
151 raise HTTPException(
152 status_code=500,
153 detail=f"Failed to get artifact '{artifact_name}': {artifact_result['message']}",
154 )
155 except Exception as err:
156 logger.error(f"Failed to get artifact '{artifact_name}': {err}")
157 raise HTTPException(
158 status_code=500,
159 detail=f"Failed to get artifact '{artifact_name}': {err}",
160 )
161
162
163 async def validate_artifact_parameters(
164 artifact_name: str,
165 provided_parameters: Optional[Dict[str, Union[str, List[ParameterKeyValue]]]],
166 ) -> None:
167 """
168 Validates that the provided parameters match the artifact's expected parameters.
169
170 Args:
171 artifact_name (str): The name of the artifact to validate against.
172 provided_parameters (Optional[Dict]): The parameters provided in the request.
173
174 Raises:
175 HTTPException: If any provided parameter is not valid for the artifact.
176 """
177 if not provided_parameters:
178 return # No parameters to validate
179
180 # Fetch the artifact details
181 artifact_response = await get_artifact_by_name(artifact_name)
182
183 if not artifact_response.artifacts or len(artifact_response.artifacts) == 0:
184 raise HTTPException(
185 status_code=404,
186 detail=f"Artifact {artifact_name} not found",
187 )
188
189 artifact = artifact_response.artifacts[0]
190
191 # If the artifact has no parameters defined, reject any provided parameters
192 if not artifact.parameters:
193 raise HTTPException(
194 status_code=400,
195 detail=f"Artifact {artifact_name} does not accept any parameters",
196 )
197
198 # Get valid parameter names from the artifact
199 valid_param_names = {param.name for param in artifact.parameters}
200
201 logger.info(f"Valid parameters for artifact {artifact_name}: {valid_param_names}")
202 logger.info(f"Provided parameters for artifact {artifact_name}: {provided_parameters}")
203
204 # Check if provided parameters are valid
205 # Handle both direct key-value pairs and the 'env' list format
206 if isinstance(provided_parameters, dict):
207 if "env" in provided_parameters and isinstance(provided_parameters["env"], list):
208 # Handle env list format
209 for param_pair in provided_parameters["env"]:
210 # Check if it's a ParameterKeyValue model or a dict
211 if isinstance(param_pair, ParameterKeyValue):
212 param_name = param_pair.key
213 elif isinstance(param_pair, dict) and "key" in param_pair:
214 param_name = param_pair["key"]
215 else:
216 continue
217
218 if param_name not in valid_param_names:
219 raise HTTPException(
220 status_code=400,
221 detail=f"Parameter '{param_name}' is not valid for artifact {artifact_name}. Valid parameters: {', '.join(sorted(valid_param_names))}",
222 )
223 else:
224 # Handle direct key-value format
225 for param_name in provided_parameters.keys():
226 if param_name not in valid_param_names:
227 raise HTTPException(
228 status_code=400,
229 detail=f"Parameter '{param_name}' is not valid for artifact {artifact_name}. Valid parameters: {', '.join(sorted(valid_param_names))}",
230 )
231
232 logger.info(f"All provided parameters are valid for artifact {artifact_name}")
233
234
235 async def get_artifact_parameters_by_prefix_service(
236 artifact_name: str,
237 parameter_prefix: str,
238 ) -> ArtifactParametersResponse:
239 """
240 Get parameters from a specific artifact that match a given prefix.
241
242 Args:
243 artifact_name (str): The name of the artifact to retrieve parameters from.
244 parameter_prefix (str): The prefix to filter parameters by.
245
246 Returns:
247 ArtifactParametersResponse: A response containing matching parameters.
248 """
249 logger.info(f"Fetching parameters with prefix '{parameter_prefix}' from artifact '{artifact_name}'")
250
251 try:
252 # First, get the artifact with its parameters
253 artifact_response = await get_artifact_by_name(artifact_name)
254
255 if not artifact_response.success or not artifact_response.artifacts:
256 return ArtifactParametersResponse(
257 success=False,
258 message=f"Artifact '{artifact_name}' not found",
259 artifact_name=artifact_name,
260 parameter_prefix=parameter_prefix,
261 matching_parameters=[],
262 total_matches=0,
263 )
264
265 artifact = artifact_response.artifacts[0]
266
267 # Filter parameters by prefix
268 matching_parameters = []
269 if artifact.parameters:
270 for param in artifact.parameters:
271 if param.name.startswith(parameter_prefix):
272 matching_parameters.append(param)
273
274 # Sort the matching parameters for consistent ordering
275 matching_parameters.sort(key=lambda x: x.name)
276
277 total_matches = len(matching_parameters)
278
279 if total_matches == 0:
280 message = f"No parameters found matching prefix '{parameter_prefix}' in artifact '{artifact_name}'"
281 elif total_matches == 1:
282 message = f"Found 1 parameter matching prefix '{parameter_prefix}' in artifact '{artifact_name}'"
283 else:
284 message = f"Found {total_matches} parameters matching prefix '{parameter_prefix}' in artifact '{artifact_name}'"
285
286 logger.info(message)
287
288 return ArtifactParametersResponse(
289 success=True,
290 message=message,
291 artifact_name=artifact_name,
292 parameter_prefix=parameter_prefix,
293 matching_parameters=matching_parameters,
294 total_matches=total_matches,
295 )
296
297 except Exception as err:
298 error_message = f"Failed to get parameters with prefix '{parameter_prefix}' from artifact '{artifact_name}': {err}"
299 logger.error(error_message)
300 raise HTTPException(
301 status_code=500,
302 detail=error_message,
303 )
304
305
306 # async def run_artifact_collection(
307 # collect_artifact_body: CollectArtifactBody,
308 # ) -> CollectArtifactResponse:
309 # """
310 # Run an artifact collection on a client with optional parameters.
311
312 # Args:
313 # collect_artifact_body: The body of the request with optional parameters.
314
315 # Returns:
316 # CollectArtifactResponse: A dictionary containing the success status, message and results.
317 # """
318 # velociraptor_service = await UniversalService.create("Velociraptor")
319 # try:
320 # # Build the query dynamically based on whether parameters are provided
321 # parameters = getattr(collect_artifact_body, "parameters", None)
322
323 # if parameters:
324 # # Velociraptor expects parameters in a very specific format
325 # # For the "env" parameter, we need to construct a dict
326 # if "env" in parameters and isinstance(parameters["env"], list):
327 # env_dict = {}
328 # for item in parameters["env"]:
329 # env_dict[item.key] = item.value
330
331 # # Format the query with proper VQL syntax
332 # query = create_query(
333 # f"SELECT collect_client("
334 # f"org_id='{collect_artifact_body.velociraptor_org}', "
335 # f"client_id='{collect_artifact_body.velociraptor_id}', "
336 # f"artifacts=['{collect_artifact_body.artifact_name}'], "
337 # f"env=dict(",
338 # )
339
340 # # Add each environment variable as a key-value pair
341 # env_parts = []
342 # for key, value in env_dict.items():
343 # # Escape any single quotes in the values
344 # escaped_value = value.replace("'", "\\'")
345 # env_parts.append(f"`{key}`='{escaped_value}'")
346
347 # query += ", ".join(env_parts)
348 # query += ")) FROM scope()"
349 # else:
350 # # Handle other types of parameters
351 # query = create_query(
352 # f"SELECT collect_client("
353 # f"org_id='{collect_artifact_body.velociraptor_org}', "
354 # f"client_id='{collect_artifact_body.velociraptor_id}', "
355 # f"artifacts=['{collect_artifact_body.artifact_name}']",
356 # )
357
358 # # Add other parameters if needed
359 # for param_key, param_value in parameters.items():
360 # if isinstance(param_value, str):
361 # query += f", `{param_key}`='{param_value}'"
362
363 # query += ") FROM scope()"
364 # else:
365 # # Original query without parameters
366 # query = create_query(
367 # f"SELECT collect_client("
368 # f"org_id='{collect_artifact_body.velociraptor_org}', "
369 # f"client_id='{collect_artifact_body.velociraptor_id}', "
370 # f"artifacts=['{collect_artifact_body.artifact_name}']) "
371 # f"FROM scope()",
372 # )
373
374 # logger.info(f"Running artifact collection with query: {query}")
375 # flow = velociraptor_service.execute_query(query, org_id=collect_artifact_body.velociraptor_org)
376 # logger.info(f"Successfully ran artifact collection on {flow}")
377
378 # # Check if results are available
379 # if not flow.get("results") or len(flow["results"]) == 0:
380 # logger.error("No results returned from query execution")
381 # raise HTTPException(
382 # status_code=500,
383 # detail="Query execution did not return any results",
384 # )
385
386 # # Instead of relying on get_artifact_key, extract the flow_id directly from results
387 # # by checking all keys in the first result for a flow_id
388 # result_dict = flow["results"][0]
389 # flow_id = None
390
391 # # Look for any key that has a flow_id in its value
392 # for key, value in result_dict.items():
393 # if isinstance(value, dict) and "flow_id" in value:
394 # flow_id = value["flow_id"]
395 # logger.debug(f"Found flow_id {flow_id} in key: {key}")
396 # break
397
398 # if not flow_id:
399 # logger.error(f"Could not find flow_id in results: {result_dict}")
400 # raise HTTPException(
401 # status_code=500,
402 # detail="Failed to extract flow ID from results",
403 # )
404
405 # logger.info(f"Extracted flow_id: {flow_id}")
406
407 # completed = velociraptor_service.watch_flow_completion(flow_id, org_id=collect_artifact_body.velociraptor_org)
408 # logger.info(f"Successfully watched flow completion on {completed}")
409
410 # results = velociraptor_service.read_collection_results(
411 # client_id=collect_artifact_body.velociraptor_id,
412 # flow_id=flow_id,
413 # org_id=collect_artifact_body.velociraptor_org,
414 # artifact=collect_artifact_body.artifact_name,
415 # )
416
417 # logger.info(f"Successfully read collection results on {results}")
418
419 # return CollectArtifactResponse(
420 # success=results["success"],
421 # message=results["message"],
422 # results=results["results"],
423 # )
424 # except HTTPException as he: # Catch HTTPException separately to propagate the original message
425 # logger.error(
426 # f"HTTPException while running artifact collection on {collect_artifact_body}: {he.detail}",
427 # )
428 # raise he
429 # except Exception as err:
430 # logger.error(
431 # f"Failed to run artifact collection on {collect_artifact_body}: {err}",
432 # )
433 # raise HTTPException(
434 # status_code=500,
435 # detail=f"Failed to run artifact collection on {collect_artifact_body}: {err}",
436 # )
437
438 async def run_artifact_collection(
439 collect_artifact_body: CollectArtifactBody,
440 session: AsyncSession,
441 ) -> CollectArtifactResponse:
442 """
443 Run an artifact collection on a client with optional parameters and upload results to MinIO.
444
445 Args:
446 collect_artifact_body: The body of the request with optional parameters.
447 session: Database session for storing metadata.
448
449 Returns:
450 CollectArtifactResponse: A dictionary containing the success status, message and results.
451 """
452 from sqlalchemy import select
453
454 from app.db.universal_models import AgentDataStore
455 from app.db.universal_models import Agents
456
457 velociraptor_service = await UniversalService.create("Velociraptor")
458 try:
459 # Get agent details from database
460 result = await session.execute(
461 select(Agents).where(Agents.velociraptor_id == collect_artifact_body.velociraptor_id),
462 )
463 agent = result.scalars().first()
464
465 if not agent:
466 raise HTTPException(
467 status_code=404,
468 detail=f"Agent with velociraptor_id {collect_artifact_body.velociraptor_id} not found",
469 )
470
471 # Build the query dynamically based on whether parameters are provided
472 parameters = getattr(collect_artifact_body, "parameters", None)
473
474 if parameters:
475 # Velociraptor expects parameters in a very specific format
476 # For the "env" parameter, we need to construct a dict
477 if "env" in parameters and isinstance(parameters["env"], list):
478 env_dict = {}
479 for item in parameters["env"]:
480 env_dict[item.key] = item.value
481
482 # Format the query with proper VQL syntax
483 query = create_query(
484 f"SELECT collect_client("
485 f"org_id='{collect_artifact_body.velociraptor_org}', "
486 f"client_id='{collect_artifact_body.velociraptor_id}', "
487 f"artifacts=['{collect_artifact_body.artifact_name}'], "
488 f"env=dict(",
489 )
490
491 # Add each environment variable as a key-value pair
492 env_parts = []
493 for key, value in env_dict.items():
494 # Escape any single quotes in the values
495 escaped_value = value.replace("'", "\\'")
496 env_parts.append(f"`{key}`='{escaped_value}'")
497
498 query += ", ".join(env_parts)
499 query += ")) FROM scope()"
500 else:
501 # Handle other types of parameters
502 query = create_query(
503 f"SELECT collect_client("
504 f"org_id='{collect_artifact_body.velociraptor_org}', "
505 f"client_id='{collect_artifact_body.velociraptor_id}', "
506 f"artifacts=['{collect_artifact_body.artifact_name}']",
507 )
508
509 # Add other parameters if needed
510 for param_key, param_value in parameters.items():
511 if isinstance(param_value, str):
512 query += f", `{param_key}`='{param_value}'"
513
514 query += ") FROM scope()"
515 else:
516 # Original query without parameters
517 query = create_query(
518 f"SELECT collect_client("
519 f"org_id='{collect_artifact_body.velociraptor_org}', "
520 f"client_id='{collect_artifact_body.velociraptor_id}', "
521 f"artifacts=['{collect_artifact_body.artifact_name}']) "
522 f"FROM scope()",
523 )
524
525 logger.info(f"Running artifact collection with query: {query}")
526 flow = velociraptor_service.execute_query(query, org_id=collect_artifact_body.velociraptor_org)
527 logger.info(f"Successfully ran artifact collection on {flow}")
528
529 # Check if results are available
530 if not flow.get("results") or len(flow["results"]) == 0:
531 logger.error("No results returned from query execution")
532 raise HTTPException(
533 status_code=500,
534 detail="Query execution did not return any results",
535 )
536
537 # Instead of relying on get_artifact_key, extract the flow_id directly from results
538 # by checking all keys in the first result for a flow_id
539 result_dict = flow["results"][0]
540 flow_id = None
541
542 # Look for any key that has a flow_id in its value
543 for key, value in result_dict.items():
544 if isinstance(value, dict) and "flow_id" in value:
545 flow_id = value["flow_id"]
546 logger.debug(f"Found flow_id {flow_id} in key: {key}")
547 break
548
549 if not flow_id:
550 logger.error(f"Could not find flow_id in results: {result_dict}")
551 raise HTTPException(
552 status_code=500,
553 detail="Failed to extract flow ID from results",
554 )
555
556 logger.info(f"Extracted flow_id: {flow_id}")
557
558 completed = velociraptor_service.watch_flow_completion(flow_id, org_id=collect_artifact_body.velociraptor_org)
559 logger.info(f"Successfully watched flow completion on {completed}")
560
561 # Only read collection results if data_store_only is False
562 results = None
563 if not collect_artifact_body.data_store_only:
564 results = velociraptor_service.read_collection_results(
565 client_id=collect_artifact_body.velociraptor_id,
566 flow_id=flow_id,
567 org_id=collect_artifact_body.velociraptor_org,
568 artifact=collect_artifact_body.artifact_name,
569 )
570 logger.info(f"Successfully read collection results on {results}")
571
572 # Fetch the collected file from filestore and upload to MinIO
573 file_data = None
574 try:
575 logger.info("Fetching collected file from filestore and uploading to MinIO")
576 file_data = await fetch_file_from_filestore(
577 client_id=collect_artifact_body.velociraptor_id,
578 flow_id=flow_id,
579 org_id=collect_artifact_body.velociraptor_org,
580 agent_id=agent.agent_id,
581 )
582
583 # Save metadata to database if file upload was successful
584 if file_data.get("success"):
585 agent_data_store = AgentDataStore(
586 agent_id=agent.agent_id,
587 velociraptor_id=collect_artifact_body.velociraptor_id,
588 artifact_name=collect_artifact_body.artifact_name,
589 flow_id=flow_id,
590 bucket_name=file_data["bucket_name"],
591 object_key=file_data["object_key"],
592 file_name=file_data["file_name"],
593 content_type=file_data.get("content_type", "application/zip"),
594 file_size=file_data["file_size"],
595 file_hash=file_data["file_hash"],
596 collection_time=datetime.utcnow(),
597 status="completed",
598 )
599 session.add(agent_data_store)
600 await session.commit()
601 await session.refresh(agent_data_store)
602
603 logger.info(f"Saved artifact metadata to database with ID {agent_data_store.id}")
604 except Exception as file_err:
605 logger.warning(f"Failed to upload file to MinIO, but artifact collection succeeded: {file_err}")
606 # Continue execution even if file upload fails
607
608 # Build response based on data_store_only flag
609 if collect_artifact_body.data_store_only:
610 return CollectArtifactResponse(
611 success=True,
612 message="Artifact collected and stored successfully. Results not retrieved.",
613 results=None,
614 file_info=file_data if file_data and file_data.get("success") else None,
615 )
616 else:
617 return CollectArtifactResponse(
618 success=results["success"],
619 message=results["message"],
620 results=results["results"],
621 file_info=file_data if file_data and file_data.get("success") else None,
622 )
623 except HTTPException as he: # Catch HTTPException separately to propagate the original message
624 logger.error(
625 f"HTTPException while running artifact collection on {collect_artifact_body}: {he.detail}",
626 )
627 raise he
628 except Exception as err:
629 logger.error(
630 f"Failed to run artifact collection on {collect_artifact_body}: {err}",
631 )
632 raise HTTPException(
633 status_code=500,
634 detail=f"Failed to run artifact collection on {collect_artifact_body}: {err}",
635 )
636
637
638 async def run_file_collection(
639 collect_artifact_body: CollectFileBody,
640 session: AsyncSession, # Add this parameter
641 ) -> CollectArtifactResponse:
642 """
643 Run an artifact collection on a client and store the result in MinIO.
644 """
645 from sqlalchemy import select
646
647 from app.db.universal_models import AgentDataStore
648 from app.db.universal_models import Agents
649
650 velociraptor_service = await UniversalService.create("Velociraptor")
651
652 try:
653 # Get agent details from database
654 result = await session.execute(
655 select(Agents).where(Agents.velociraptor_id == collect_artifact_body.velociraptor_id),
656 )
657 agent = result.scalars().first()
658
659 if not agent:
660 raise HTTPException(
661 status_code=404,
662 detail=f"Agent with velociraptor_id {collect_artifact_body.velociraptor_id} not found",
663 )
664
665 # Build the query with proper VQL syntax
666 query = create_query(
667 f"SELECT collect_client("
668 f"org_id='{collect_artifact_body.velociraptor_org}', "
669 f"client_id='{collect_artifact_body.velociraptor_id}', "
670 f"artifacts=['{collect_artifact_body.artifact_name}'], "
671 f"spec=dict(`{collect_artifact_body.artifact_name}`=dict("
672 f"`collectionSpec`='{collect_artifact_body.file}', "
673 f"`Root`='{collect_artifact_body.root_disk}'"
674 f"))) "
675 f"FROM scope()",
676 )
677
678 logger.info(f"Query: {query}")
679 flow = velociraptor_service.execute_query(query, org_id=collect_artifact_body.velociraptor_org)
680 logger.info(f"Successfully ran artifact collection on {flow}")
681
682 # Check if results are available
683 if not flow.get("results") or len(flow["results"]) == 0:
684 logger.error("No results returned from query execution")
685 raise HTTPException(
686 status_code=500,
687 detail="Query execution did not return any results",
688 )
689
690 # Extract flow_id from results
691 result_dict = flow["results"][0]
692 flow_id = None
693
694 for key, value in result_dict.items():
695 if isinstance(value, dict) and "flow_id" in value:
696 flow_id = value["flow_id"]
697 logger.debug(f"Found flow_id {flow_id} in key: {key}")
698 break
699
700 if not flow_id:
701 logger.error(f"Could not find flow_id in results: {result_dict}")
702 raise HTTPException(
703 status_code=500,
704 detail="Failed to extract flow ID from results",
705 )
706
707 logger.info(f"Extracted flow_id: {flow_id}")
708
709 completed = velociraptor_service.watch_flow_completion(flow_id, org_id=collect_artifact_body.velociraptor_org)
710 logger.info(f"Successfully watched flow completion on {completed}")
711
712 results = velociraptor_service.read_collection_results(
713 client_id=collect_artifact_body.velociraptor_id,
714 flow_id=flow_id,
715 org_id=collect_artifact_body.velociraptor_org,
716 artifact=collect_artifact_body.artifact_name,
717 )
718
719 logger.info("Successfully read collection results")
720
721 # Fetch the collected file from filestore and upload to MinIO
722 logger.info("Fetching collected file from filestore and uploading to MinIO")
723 file_data = await fetch_file_from_filestore(
724 client_id=collect_artifact_body.velociraptor_id,
725 flow_id=flow_id,
726 org_id=collect_artifact_body.velociraptor_org,
727 agent_id=agent.agent_id,
728 )
729
730 # Save metadata to database
731 if file_data.get("success"):
732 agent_data_store = AgentDataStore(
733 agent_id=agent.agent_id,
734 velociraptor_id=collect_artifact_body.velociraptor_id,
735 artifact_name=collect_artifact_body.artifact_name,
736 flow_id=flow_id,
737 bucket_name=file_data["bucket_name"],
738 object_key=file_data["object_key"],
739 file_name=file_data["file_name"],
740 content_type=file_data.get("content_type", "application/zip"),
741 file_size=file_data["file_size"],
742 file_hash=file_data["file_hash"],
743 collection_time=datetime.utcnow(),
744 status="completed",
745 )
746 session.add(agent_data_store)
747 await session.commit()
748 await session.refresh(agent_data_store)
749
750 logger.info(f"Saved artifact metadata to database with ID {agent_data_store.id}")
751
752 return CollectArtifactResponse(
753 success=results["success"],
754 message=results["message"],
755 results=results["results"],
756 file_info=file_data if file_data.get("success") else None,
757 )
758
759 except HTTPException as he:
760 logger.error(f"HTTPException while running artifact collection: {he.detail}")
761 raise he
762 except Exception as err:
763 logger.error(f"Failed to run artifact collection: {err}")
764 raise HTTPException(
765 status_code=500,
766 detail=f"Failed to run artifact collection: {err}",
767 )
768
769
770 async def fetch_file_from_filestore(
771 client_id: str,
772 flow_id: str,
773 org_id: str,
774 agent_id: str, # Add this parameter
775 hostname: Optional[str] = None,
776 password: Optional[str] = None,
777 format: str = "json",
778 expand_sparse: bool = False,
779 ) -> dict:
780 """
781 Fetch a file from Velociraptor's filestore and save it to MinIO.
782 First creates a download pack for the flow, then fetches and uploads to MinIO.
783 """
784 import base64
785 import os
786
787 from app.data_store.data_store_operations import upload_agent_artifact_file
788
789 velociraptor_service = await UniversalService.create("Velociraptor")
790
791 try:
792 # If hostname is not provided, fetch it from client metadata
793 if not hostname:
794 logger.info(f"Fetching hostname for client {client_id}")
795 query = create_query(
796 f"SELECT os_info.hostname AS Hostname "
797 f"FROM clients(client_id='{client_id}')",
798 )
799 client_info = velociraptor_service.execute_query(query, org_id=org_id)
800
801 if client_info.get("results") and len(client_info["results"]) > 0:
802 hostname = client_info["results"][0].get("Hostname", client_id)
803 else:
804 hostname = client_id
805 logger.warning(f"Could not fetch hostname, using client_id: {client_id}")
806
807 # Step 1: Create the flow download
808 logger.info(f"Creating flow download for client {client_id}, flow {flow_id}")
809
810 download_query_parts = [
811 f"client_id='{client_id}'",
812 f"flow_id='{flow_id}'",
813 "wait=true",
814 f"format='{format}'",
815 f"expand_sparse={str(expand_sparse).lower()}",
816 ]
817
818 if password:
819 download_query_parts.append(f"password='{password}'")
820
821 zip_filename = f"{hostname}-{client_id}-{flow_id}.zip"
822 download_query_parts.append(f"name='{hostname}-{client_id}-{flow_id}'")
823
824 create_download_query = create_query(
825 f"SELECT create_flow_download({', '.join(download_query_parts)}) "
826 f"FROM scope()",
827 )
828
829 logger.info(f"Create download query: {create_download_query}")
830 download_result = velociraptor_service.execute_query(create_download_query, org_id=org_id)
831
832 if not download_result.get("success"):
833 raise HTTPException(
834 status_code=500,
835 detail=f"Failed to create flow download: {download_result.get('message')}",
836 )
837
838 logger.info("Flow download created successfully")
839
840 # Step 2: Fetch the file from Velociraptor
841 vfs_path = f"downloads/{client_id}/{flow_id}/{zip_filename}"
842 logger.info(f"Fetching file from VFS path: {vfs_path}")
843
844 # Save to temporary location first
845 temp_file_path = os.path.join(os.getcwd(), zip_filename)
846 offset = 0
847 chunk_size = 1024 * 1024 # 1MB chunks
848 total_bytes = 0
849
850 with open(temp_file_path, 'wb') as f:
851 while True:
852 query = create_query(
853 f"SELECT base64encode(string=read_file("
854 f"accessor='fs', "
855 f"filename='/{vfs_path}', "
856 f"offset={offset}, "
857 f"length={chunk_size})) AS Data "
858 f"FROM scope()",
859 )
860
861 result = velociraptor_service.execute_query(query, org_id=org_id)
862
863 if not result.get("success"):
864 raise HTTPException(
865 status_code=500,
866 detail=f"Failed to fetch file chunk: {result.get('message')}",
867 )
868
869 if not result.get("results") or len(result["results"]) == 0:
870 break
871
872 data = result["results"][0].get("Data")
873 if not data:
874 break
875
876 try:
877 decoded_data = base64.b64decode(data)
878 if len(decoded_data) == 0:
879 break
880
881 f.write(decoded_data)
882 chunk_bytes = len(decoded_data)
883 total_bytes += chunk_bytes
884 offset += chunk_bytes
885
886 logger.debug(f"Fetched {chunk_bytes} bytes, total: {total_bytes}")
887
888 if chunk_bytes < chunk_size:
889 break
890 except Exception as e:
891 logger.error(f"Failed to decode chunk: {e}")
892 break
893
894 if total_bytes == 0:
895 logger.warning(f"No file data found at path: {vfs_path}")
896 if os.path.exists(temp_file_path):
897 os.remove(temp_file_path)
898 return {
899 "success": False,
900 "message": f"No file found at path: {vfs_path}",
901 "file_path": None,
902 "file_size": 0,
903 }
904
905 logger.info(f"Successfully fetched file of size {total_bytes} bytes")
906
907 # Step 3: Upload to MinIO
908 logger.info(f"Uploading artifact file to MinIO for agent {agent_id}")
909 upload_result = await upload_agent_artifact_file(
910 agent_id=agent_id,
911 flow_id=flow_id,
912 file_path=temp_file_path,
913 file_name=zip_filename,
914 )
915
916 # Remove temporary file
917 os.remove(temp_file_path)
918 logger.info(f"Removed temporary file {temp_file_path}")
919
920 return {
921 "success": True,
922 "message": "Successfully fetched file from Velociraptor and uploaded to MinIO",
923 **upload_result,
924 }
925
926 except HTTPException as he:
927 logger.error(f"HTTPException while fetching file: {he.detail}")
928 raise he
929 except Exception as err:
930 logger.error(f"Failed to fetch file from filestore: {err}")
931 raise HTTPException(
932 status_code=500,
933 detail=f"Failed to fetch file from filestore: {err}",
934 )
935
936
937 async def run_remote_command(run_command_body: RunCommandBody) -> RunCommandResponse:
938 """
939 Run a remote command on a client.
940
941 Args:
942 run_analyzer_body (RunAnalyzerBody): The body of the request.
943
944 Returns:
945 RunAnalyzerResponse: A dictionary containing the success status and a message.
946 """
947 velociraptor_service = await UniversalService.create("Velociraptor")
948 try:
949 run_command_body.artifact_name = run_command_body.artifact_name.value
950 logger.info(f"Running remote command on {run_command_body}")
951 query = create_query(
952 (
953 f"SELECT collect_client(org_id='{run_command_body.velociraptor_org}', client_id='{run_command_body.velociraptor_id}', "
954 f"urgent=true, artifacts=['{run_command_body.artifact_name}'], "
955 f"env=dict(Command='{run_command_body.command}')) "
956 "FROM scope()"
957 ),
958 )
959 flow = velociraptor_service.execute_query(query, org_id=run_command_body.velociraptor_org)
960 logger.info(f"Successfully ran artifact collection on {flow}")
961
962 artifact_key = get_artifact_key(analyzer_body=run_command_body)
963
964 flow_id = flow["results"][0][artifact_key]["flow_id"]
965 logger.info(f"Extracted flow_id: {flow_id}")
966
967 completed = velociraptor_service.watch_flow_completion(flow_id, org_id=run_command_body.velociraptor_org)
968 logger.info(f"Successfully watched flow completion on {completed}")
969
970 results = velociraptor_service.read_collection_results(
971 client_id=run_command_body.velociraptor_id,
972 flow_id=flow_id,
973 org_id=run_command_body.velociraptor_org,
974 artifact=run_command_body.artifact_name,
975 )
976
977 logger.info(f"Successfully read collection results on {results}")
978
979 return RunCommandResponse(
980 success=results["success"],
981 message=results["message"],
982 results=results["results"],
983 )
984 except Exception as err:
985 logger.error(f"Failed to run artifact collection on {run_command_body}: {err}")
986 raise HTTPException(
987 status_code=500,
988 detail=f"Failed to run artifact collection on {run_command_body}: {err}",
989 )
990
991
992 async def quarantine_host(quarantine_body: QuarantineBody) -> QuarantineResponse:
993 """
994 Quarantine a host.
995
996 Args:
997 quarantine_body (QuarantineBody): The body of the request.
998
999 Returns:
1000 QuarantineResponse: A dictionary containing the success status and a message.
1001 """
1002 velociraptor_service = await UniversalService.create("Velociraptor")
1003 try:
1004 quarantine_body.artifact_name = quarantine_body.artifact_name.value
1005 quarantine_body.action = quarantine_body.action.value
1006 if quarantine_body.action == "quarantine":
1007 query = create_query(
1008 (
1009 f'SELECT collect_client(org_id="{quarantine_body.velociraptor_org}", client_id="{quarantine_body.velociraptor_id}", '
1010 f'artifacts=["{quarantine_body.artifact_name}"], '
1011 f"spec=dict(`{quarantine_body.artifact_name}`=dict())) "
1012 "FROM scope()"
1013 ),
1014 )
1015 else:
1016 query = create_query(
1017 (
1018 f'SELECT collect_client(org_id="{quarantine_body.velociraptor_org}", client_id="{quarantine_body.velociraptor_id}", '
1019 f'artifacts=["{quarantine_body.artifact_name}"], '
1020 f'spec=dict(`{quarantine_body.artifact_name}`=dict(`RemovePolicy`="Y"))) '
1021 "FROM scope()"
1022 ),
1023 )
1024 flow = velociraptor_service.execute_query(query, org_id=quarantine_body.velociraptor_org)
1025 logger.info(f"Successfully ran artifact collection on {flow}")
1026
1027 artifact_key = get_artifact_key(analyzer_body=quarantine_body)
1028
1029 flow_id = flow["results"][0][artifact_key]["flow_id"]
1030 logger.info(f"Extracted flow_id: {flow_id}")
1031
1032 completed = velociraptor_service.watch_flow_completion(flow_id, org_id=quarantine_body.velociraptor_org)
1033 logger.info(f"Successfully watched flow completion on {completed}")
1034
1035 results = velociraptor_service.read_collection_results(
1036 client_id=quarantine_body.velociraptor_id,
1037 flow_id=flow_id,
1038 org_id=quarantine_body.velociraptor_org,
1039 artifact=quarantine_body.artifact_name,
1040 )
1041
1042 logger.info(f"Successfully read collection results on {results}")
1043
1044 return QuarantineResponse(
1045 success=results["success"],
1046 message=results["message"],
1047 results=results["results"],
1048 )
1049 except Exception as err:
1050 logger.error(f"Failed to run artifact collection on {quarantine_body}: {err}")
1051 raise HTTPException(
1052 status_code=500,
1053 detail=f"Failed to run artifact collection on {quarantine_body}: {err}",
1054 )
1055
1056
1057 ################# ! ARTIFACT RECOMMENDATION ! #################
1058 async def post_to_copilot_ai_module(data: ArtifactReccomendationRequest) -> ArtifactReccomendationResponse:
1059 """
1060 Send a POST request to the copilot-ai-module Docker container.
1061
1062 Args:
1063 data (ArtifactReccomendationRequest): The data to send to the copilot-ai-module Docker container.
1064 """
1065 logger.info(f"Sending POST request to http://copilot-ai-module/velociraptor-artifact-recommendation with data: {data.model_dump()}")
1066 # raise HTTPException(status_code=501, detail="Not Implemented Yet")
1067 async with httpx.AsyncClient() as client:
1068 data = await client.post(
1069 "http://copilot-ai-module/velociraptor-artifact-recommendation",
1070 json=data.model_dump(),
1071 timeout=120,
1072 )
1073 response_data = data.json()
1074
1075 if not response_data.get("success"):
1076 raise HTTPException(
1077 status_code=400,
1078 detail=response_data.get("message", "Request to copilot-ai-module was not successful"),
1079 )
1080 return ArtifactReccomendationResponse(**data.json())