@cryptotaxi247 / CoPilot / commits / 2f63ca91

remove unused sigma (#812)

taylor_socfortress committed Apr 24, 2026 at 12:18 UTC 2f63ca917a129b58276ab1f33d441af1bdeaa38e
13 files changed -2287
backend/app/connectors/wazuh_indexer/routes/sigma.py deleted
-545
@@ -1,545 +0,0 @@
1 -import os
2 -from datetime import datetime
3 -
4 -from fastapi import APIRouter
5 -from fastapi import Depends
6 -from fastapi import File
7 -from fastapi import HTTPException
8 -from fastapi import Query
9 -from fastapi import Security
10 -from fastapi import UploadFile
11 -from loguru import logger
12 -from sqlalchemy.ext.asyncio import AsyncSession
13 -
14 -from app.auth.utils import AuthHandler
15 -from app.connectors.wazuh_indexer.schema.sigma import ActivateSigmaQueryResponse
16 -from app.connectors.wazuh_indexer.schema.sigma import BulkUploadToDBResponse
17 -from app.connectors.wazuh_indexer.schema.sigma import CreateSigmaQuery
18 -from app.connectors.wazuh_indexer.schema.sigma import DeactivateSigmaQueryResponse
19 -from app.connectors.wazuh_indexer.schema.sigma import DeleteSigmaQueryResponse
20 -from app.connectors.wazuh_indexer.schema.sigma import DownloadSigmaRulesRequest
21 -from app.connectors.wazuh_indexer.schema.sigma import RunActiveSigmaQueries
22 -from app.connectors.wazuh_indexer.schema.sigma import SigmaQueryOutResponse
23 -from app.connectors.wazuh_indexer.schema.sigma import SigmaRuleUploadRequest
24 -from app.connectors.wazuh_indexer.schema.sigma import UpdateSigmaActive
25 -from app.connectors.wazuh_indexer.schema.sigma import UpdateSigmaTimeInterval
26 -from app.connectors.wazuh_indexer.services.sigma.execute_query import execute_query
27 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
28 - add_sigma_queries_to_db,
29 -)
30 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
31 - create_sigma_query,
32 -)
33 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
34 - delete_sigma_rule,
35 -)
36 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
37 - get_sigma_query_by_id,
38 -)
39 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
40 - list_active_sigma_queries,
41 -)
42 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
43 - list_inactive_sigma_queries,
44 -)
45 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
46 - list_sigma_queries,
47 -)
48 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
49 - parse_time_interval,
50 -)
51 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
52 - process_sigma_file,
53 -)
54 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
55 - set_sigma_query_active,
56 -)
57 -from app.connectors.wazuh_indexer.services.sigma.sigma_db_operations import (
58 - update_sigma_time_interval,
59 -)
60 -from app.connectors.wazuh_indexer.services.sigma.sigma_download import (
61 - download_and_extract_zip,
62 -)
63 -from app.connectors.wazuh_indexer.services.sigma.sigma_download import (
64 - keep_only_folder_directory,
65 -)
66 -from app.db.db_session import get_db
67 -
68 -wazuh_indexer_sigma_router = APIRouter()
69 -
70 -
71 -def save_file(file: UploadFile, file_path: str):
72 - """
73 - Save the uploaded file to the specified path.
74 -
75 - Args:
76 - file (UploadFile): The uploaded file.
77 - file_path (str): The path to save the file.
78 - """
79 - try:
80 - with open(file_path, "wb") as f:
81 - f.write(file.file.read())
82 - logger.info(f"File saved to {file_path}")
83 - except Exception as e:
84 - logger.error(f"Error saving file: {e}")
85 - raise HTTPException(status_code=500, detail="An error occurred while saving the file.")
86 -
87 -
88 -def cleanup_file(file_path: str):
89 - """
90 - Remove the file from the specified path.
91 -
92 - Args:
93 - file_path (str): The path of the file to remove.
94 - """
95 - try:
96 - if os.path.exists(file_path):
97 - os.remove(file_path)
98 - logger.info(f"File {file_path} removed")
99 - except Exception as e:
100 - logger.error(f"Error removing file: {e}")
101 -
102 -
103 -@wazuh_indexer_sigma_router.get("/queries/available", response_model=SigmaQueryOutResponse)
104 -async def get_sigma_queries_endpoint(
105 - db: AsyncSession = Depends(get_db),
106 -):
107 - """
108 - Retrieves a list of Sigma queries.
109 -
110 - Args:
111 - db (AsyncSession): The database session.
112 -
113 - Returns:
114 - List[SigmaQuery]: A list of Sigma queries.
115 - """
116 - return SigmaQueryOutResponse(
117 - sigma_queries=await list_sigma_queries(db),
118 - success=True,
119 - message="Successfully retrieved the Sigma queries.",
120 - )
121 -
122 -
123 -@wazuh_indexer_sigma_router.get("/queries/active", response_model=SigmaQueryOutResponse)
124 -async def get_active_sigma_queries_endpoint(
125 - db: AsyncSession = Depends(get_db),
126 -):
127 - """
128 - Retrieves a list of active Sigma queries.
129 -
130 - Args:
131 - db (AsyncSession): The database session.
132 -
133 - Returns:
134 - List[SigmaQuery]: A list of Sigma queries.
135 - """
136 - return SigmaQueryOutResponse(
137 - sigma_queries=await list_active_sigma_queries(db),
138 - success=True,
139 - message="Successfully retrieved the active Sigma queries.",
140 - )
141 -
142 -
143 -@wazuh_indexer_sigma_router.get("/queries/inactive", response_model=SigmaQueryOutResponse)
144 -async def get_inactive_sigma_queries_endpoint(
145 - db: AsyncSession = Depends(get_db),
146 -):
147 - """
148 - Retrieves a list of inactive Sigma queries.
149 -
150 - Args:
151 - db (AsyncSession): The database session.
152 -
153 - Returns:
154 - List[SigmaQuery]: A list of Sigma queries.
155 - """
156 - return SigmaQueryOutResponse(
157 - sigma_queries=await list_inactive_sigma_queries(db),
158 - success=True,
159 - message="Successfully retrieved the inactive Sigma queries.",
160 - )
161 -
162 -
163 -@wazuh_indexer_sigma_router.post("/queries/create", response_model=SigmaQueryOutResponse)
164 -async def create_sigma_query_endpoint(
165 - sigma_query: CreateSigmaQuery,
166 - db: AsyncSession = Depends(get_db),
167 -):
168 - """
169 - Creates a Sigma query.
170 -
171 - Args:
172 - db (AsyncSession): The database session.
173 - sigma_query (CreateSigmaQuery): The Sigma query creation request.
174 -
175 - Returns:
176 - SigmaQuery: The created Sigma query.
177 - """
178 - logger.info(f"Creating Sigma query: {sigma_query.dict()}")
179 - return SigmaQueryOutResponse(
180 - sigma_queries=[await create_sigma_query(sigma_query, db)],
181 - success=True,
182 - message="Successfully created the Sigma query.",
183 - )
184 -
185 -
186 -@wazuh_indexer_sigma_router.post(
187 - "/download",
188 - dependencies=[Security(AuthHandler().get_current_user)],
189 -)
190 -async def download_sigma_queries_endpoint(
191 - request: DownloadSigmaRulesRequest,
192 - db: AsyncSession = Depends(get_db),
193 -):
194 - """
195 - Downloads the Sigma queries from the Wazuh repository.
196 -
197 - Args:
198 - db (AsyncSession): The database session.
199 -
200 - Returns:
201 - SigmaQueryOutResponse: The Sigma queries response.
202 - """
203 - # Define the URL to download the Sigma queries from
204 - await download_and_extract_zip(request.url)
205 - await keep_only_folder_directory(folder=request.folder)
206 - return {"message": "Successfully downloaded the Sigma queries.", "success": True}
207 -
208 -
209 -@wazuh_indexer_sigma_router.post("/bulk-upload-to-db", response_model=BulkUploadToDBResponse)
210 -async def upload_sigma_queries_to_db_endpoint(
211 - request: SigmaRuleUploadRequest,
212 - db: AsyncSession = Depends(get_db),
213 -):
214 - """
215 - Uploads the Sigma queries to the database.
216 -
217 - Args:
218 - db (AsyncSession): The database session.
219 -
220 - Returns:
221 - SigmaQueryOutResponse: The Sigma queries response.
222 - """
223 - await add_sigma_queries_to_db(request, db)
224 - return BulkUploadToDBResponse(success=True, message="Successfully uploaded the Sigma queries to the database.")
225 -
226 -
227 -@wazuh_indexer_sigma_router.post("/activate-all-queries", response_model=ActivateSigmaQueryResponse)
228 -async def activate_all_sigma_queries_endpoint(
229 - db: AsyncSession = Depends(get_db),
230 -):
231 - """
232 - Activates all Sigma queries.
233 -
234 -
235 - Args:
236 - db (AsyncSession): The database session.
237 -
238 - Returns:
239 - ActivateSigmaQueryResponse: The Sigma queries response.
240 - """
241 - sigma_queries = await list_sigma_queries(db)
242 - enabled_queries = []
243 - for query in sigma_queries:
244 - await set_sigma_query_active(query.rule_name, True, db)
245 - enabled_queries.append(query.rule_name)
246 - return ActivateSigmaQueryResponse(
247 - success=True,
248 - message="Successfully activated all Sigma queries.",
249 - enabled_queries=enabled_queries,
250 - )
251 -
252 -
253 -@wazuh_indexer_sigma_router.post("/deactivate-all-queries", response_model=DeactivateSigmaQueryResponse)
254 -async def deactivate_all_sigma_queries_endpoint(
255 - db: AsyncSession = Depends(get_db),
256 -):
257 - """
258 - Deactivates all Sigma queries.
259 -
260 - Args:
261 - db (AsyncSession): The database session.
262 -
263 - Returns:
264 - DeactivateSigmaQueryResponse: The Sigma queries response.
265 - """
266 - sigma_queries = await list_sigma_queries(db)
267 - disabled_queries = []
268 - for query in sigma_queries:
269 - await set_sigma_query_active(query.rule_name, False, db)
270 - disabled_queries.append(query.rule_name)
271 - return DeactivateSigmaQueryResponse(
272 - success=True,
273 - message="Successfully deactivated all Sigma queries.",
274 - disabled_queries=disabled_queries,
275 - )
276 -
277 -
278 -# @wazuh_indexer_sigma_router.post("/run-active-queries", response_model=SigmaQueryOutResponse)
279 -# async def run_active_sigma_queries_endpoint(
280 -# index_name: str = Query(default="wazuh*"),
281 -# db: AsyncSession = Depends(get_db),
282 -# ):
283 -# """
284 -# Runs the active Sigma queries.
285 -
286 -# Args:
287 -# db (AsyncSession): The database session.
288 -
289 -# Returns:
290 -# SigmaQueryOutResponse: The Sigma queries response.
291 -# """
292 -# active_sigma_queries = await list_active_sigma_queries(db)
293 -# for query in active_sigma_queries:
294 -# time_interval_delta = parse_time_interval(query.time_interval)
295 -# logger.info(f"Time interval delta: {time_interval_delta}")
296 -# current_time = datetime.now()
297 -# logger.info(f"Current time: {current_time}")
298 -# logger.info(f"Last execution time: {query.last_execution_time}")
299 -
300 -# # Check if the current time is less than the last execution time
301 -# if current_time < query.last_execution_time or current_time - query.last_execution_time >= time_interval_delta:
302 -# logger.info(f"Running Sigma query: {query.rule_name}")
303 -# await execute_query(
304 -# RunActiveSigmaQueries(
305 -# query=query.rule_query,
306 -# time_interval=query.time_interval,
307 -# last_execution_time=query.last_execution_time,
308 -# rule_name=query.rule_name,
309 -# index=index_name,
310 -# ),
311 -# session=db,
312 -# )
313 -# # Update the last execution time to the current time and commit the changes
314 -# # ! Remove commented out code after testing ! #
315 -# query.last_execution_time = current_time
316 -# await db.commit()
317 -# else:
318 -# time_comparison = current_time - query.last_execution_time
319 -# logger.info(f"Time comparison: {time_comparison}")
320 -# logger.info(f"Skipping Sigma query because the time interval has not passed: {query.rule_name}")
321 -# return SigmaQueryOutResponse(
322 -# success=True,
323 -# message="Successfully ran the active Sigma queries.",
324 -# )
325 -
326 -
327 -@wazuh_indexer_sigma_router.post("/run-active-queries", response_model=SigmaQueryOutResponse)
328 -async def run_active_sigma_queries_endpoint(
329 - index_name: str = Query(default="wazuh*"),
330 - db: AsyncSession = Depends(get_db),
331 -):
332 - """
333 - Runs the active Sigma queries.
334 -
335 - Args:
336 - db (AsyncSession): The database session.
337 -
338 - Returns:
339 - SigmaQueryOutResponse: The Sigma queries response.
340 - """
341 - # ! Commenting Out for now, will revisit later if needed ! #
342 - # active_sigma_queries = await list_active_sigma_queries(db)
343 - # tasks = []
344 -
345 - # for query in active_sigma_queries:
346 - # time_interval_delta = parse_time_interval(query.time_interval)
347 - # logger.info(f"Time interval delta: {time_interval_delta}")
348 - # current_time = datetime.now()
349 - # logger.info(f"Current time: {current_time}")
350 - # logger.info(f"Last execution time: {query.last_execution_time}")
351 -
352 - # # Check if the current time is less than the last execution time
353 - # if current_time < query.last_execution_time or current_time - query.last_execution_time >= time_interval_delta:
354 - # logger.info(f"Running Sigma query: {query.rule_name}")
355 - # task = execute_query(
356 - # RunActiveSigmaQueries(
357 - # query=query.rule_query,
358 - # time_interval=query.time_interval,
359 - # last_execution_time=query.last_execution_time,
360 - # rule_name=query.rule_name,
361 - # index=index_name,
362 - # ),
363 - # session=db,
364 - # )
365 - # tasks.append(task)
366 - # # Update the last execution time to the current time
367 - # query.last_execution_time = current_time
368 -
369 - # # Run all tasks concurrently
370 - # await asyncio.gather(*tasks)
371 -
372 - # # Commit the changes to the database
373 - # await db.commit()
374 -
375 - return SigmaQueryOutResponse(
376 - success=True,
377 - message="Successfully ran the active Sigma queries.",
378 - )
379 -
380 -
381 -@wazuh_indexer_sigma_router.post("/run-single-query", response_model=SigmaQueryOutResponse)
382 -async def run_single_sigma_query_endpoint(
383 - index_name: str = Query(default="wazuh*"),
384 - rule_id: int = Query(...),
385 - db: AsyncSession = Depends(get_db),
386 -):
387 - """
388 - Runs a single Sigma query.
389 -
390 - Args:
391 - db (AsyncSession): The database session.
392 - rule_id (int): The rule ID to run.
393 -
394 - Returns:
395 - SigmaQueryOutResponse: The Sigma queries response.
396 - """
397 - query = await get_sigma_query_by_id(db=db, sigma_query_id=rule_id)
398 - time_interval_delta = parse_time_interval(query.time_interval)
399 - current_time = datetime.now()
400 - if current_time < query.last_execution_time or current_time - query.last_execution_time >= time_interval_delta:
401 - await execute_query(
402 - RunActiveSigmaQueries(
403 - query=query.rule_query,
404 - time_interval=query.time_interval,
405 - last_execution_time=query.last_execution_time,
406 - rule_name=query.rule_name,
407 - index=index_name,
408 - ),
409 - session=db,
410 - )
411 - query.last_execution_time = current_time
412 - await db.commit()
413 - else:
414 - time_comparison = current_time - query.last_execution_time
415 - logger.info(f"Time comparison: {time_comparison}")
416 - logger.info(f"Skipping Sigma query because the time interval has not passed: {query.rule_name}")
417 - return SigmaQueryOutResponse(
418 - success=True,
419 - message="Successfully ran the active Sigma queries.",
420 - )
421 -
422 -
423 -@wazuh_indexer_sigma_router.post("/upload")
424 -async def upload_sigma_queries_endpoint(
425 - file: UploadFile = File(...),
426 - db: AsyncSession = Depends(get_db),
427 -):
428 - """
429 - Uploads the Sigma queries to the database.
430 -
431 - Args:
432 - db (AsyncSession): The database session.
433 -
434 - Returns:
435 - SigmaQueryOutResponse: The Sigma queries response.
436 - """
437 - if not file.filename.endswith(".yml"):
438 - raise HTTPException(status_code=400, detail="File must be a YAML file.")
439 -
440 - file_path = f"app/connectors/wazuh_indexer/sigma_artifacts/{file.filename}"
441 -
442 - try:
443 - save_file(file, file_path)
444 - await process_sigma_file(file=file_path, db=db)
445 - except Exception as e:
446 - logger.error(f"Error processing Sigma file: {e}")
447 - raise HTTPException(status_code=500, detail="An error occurred while processing the file.")
448 - finally:
449 - cleanup_file(file_path)
450 -
451 - return {"message": "Successfully uploaded the Sigma queries to the database.", "success": True}
452 -
453 -
454 -@wazuh_indexer_sigma_router.put("/queries/set-active", response_model=SigmaQueryOutResponse)
455 -async def set_sigma_query_active_endpoint(
456 - request: UpdateSigmaActive,
457 - db: AsyncSession = Depends(get_db),
458 -):
459 - """
460 - Sets the active status of a Sigma query.
461 -
462 - Args:
463 - db (AsyncSession): The database session.
464 - rule_name (str): The rule name to set active.
465 - active (bool): The active status to set.
466 -
467 - Returns:
468 - SigmaQueryOutResponse: The Sigma queries response.
469 - """
470 - return SigmaQueryOutResponse(
471 - sigma_queries=[await set_sigma_query_active(request.rule_name, request.active, db)],
472 - success=True,
473 - message=f"Successfully set the active status of the Sigma query: {request.rule_name} to {request.active}",
474 - )
475 -
476 -
477 -@wazuh_indexer_sigma_router.put("/queries/set-time-interval", response_model=SigmaQueryOutResponse)
478 -async def set_sigma_query_time_interval_endpoint(
479 - request: UpdateSigmaTimeInterval,
480 - db: AsyncSession = Depends(get_db),
481 -):
482 - """
483 - Sets the time interval of a Sigma query.
484 -
485 - Args:
486 - db (AsyncSession): The database session.
487 - rule_name (str): The rule name to set active.
488 - time_interval (str): The time interval to set.
489 -
490 - Returns:
491 - SigmaQueryOutResponse: The Sigma queries response.
492 - """
493 - return SigmaQueryOutResponse(
494 - sigma_queries=[await update_sigma_time_interval(request.rule_name, request.time_interval, db)],
495 - success=True,
496 - message=f"Successfully set the time interval of the Sigma query: {request.rule_name} to {request.time_interval}",
497 - )
498 -
499 -
500 -@wazuh_indexer_sigma_router.delete("/queries/delete", response_model=DeleteSigmaQueryResponse)
501 -async def delete_sigma_rule_endpoint(
502 - rule_name: str = Query(...),
503 - db: AsyncSession = Depends(get_db),
504 -):
505 - """
506 - Deletes a Sigma query.
507 -
508 - Args:
509 - db (AsyncSession): The database session.
510 - rule_name (str): The rule name to delete.
511 -
512 - Returns:
513 - SigmaQueryOutResponse: The Sigma queries response.
514 - """
515 - await delete_sigma_rule(rule_name, db)
516 - return DeleteSigmaQueryResponse(
517 - deleted_queries=[rule_name],
518 - success=True,
519 - message=f"Successfully deleted the Sigma query: {rule_name}",
520 - )
521 -
522 -
523 -@wazuh_indexer_sigma_router.delete("/queries/delete-all", response_model=DeleteSigmaQueryResponse)
524 -async def delete_all_sigma_rules_endpoint(
525 - db: AsyncSession = Depends(get_db),
526 -):
527 - """
528 - Deletes all Sigma queries.
529 -
530 - Args:
531 - db (AsyncSession): The database session.
532 -
533 - Returns:
534 - DeleteSigmaQueryResponse: The Sigma queries response.
535 - """
536 - sigma_queries = await list_sigma_queries(db)
537 - deleted_queries = []
538 - for query in sigma_queries:
539 - await delete_sigma_rule(query.rule_name, db)
540 - deleted_queries.append(query.rule_name)
541 - return DeleteSigmaQueryResponse(
542 - deleted_queries=deleted_queries,
543 - success=True,
544 - message="Successfully deleted all Sigma queries.",
545 - )
backend/app/connectors/wazuh_indexer/schema/sigma.py deleted
-223
@@ -1,223 +0,0 @@
1 -import re
2 -from datetime import datetime
3 -from enum import Enum
4 -from typing import List
5 -from typing import Optional
6 -
7 -from pydantic import BaseModel
8 -from pydantic import Field
9 -from pydantic import validator
10 -
11 -
12 -class SigmaRulesLevel(str, Enum):
13 - """
14 - Represents the Sigma rules level.
15 - """
16 -
17 - high = "high"
18 - critical = "critical"
19 -
20 -
21 -class SigmaRuleUploadRequest(BaseModel):
22 - rule_levels: List[SigmaRulesLevel] = Field(
23 - ...,
24 - description="The Sigma rule levels to upload.",
25 - )
26 -
27 -
28 -class DownloadSigmaRulesRequest(BaseModel):
29 - url: str = Field(
30 - "https://github.com/SigmaHQ/sigma/releases/download/r2024-07-17/sigma_all_rules.zip",
31 - title="URL",
32 - description="The URL to download the Sigma rules from.",
33 - )
34 - folder: str = Field(
35 - "windows",
36 - title="Folder",
37 - description="The folder of Sigma rules to download.",
38 - )
39 -
40 -
41 -class BulkUploadToDBResponse(BaseModel):
42 - success: bool
43 - message: str
44 -
45 -
46 -class SigmaQueriesOut(BaseModel):
47 - """
48 - Represents the Sigma queries output.
49 - """
50 -
51 - id: int
52 - rule_name: str
53 - rule_query: str
54 - active: bool
55 - time_interval: str
56 - last_updated: Optional[datetime] = None
57 - last_execution_time: Optional[datetime] = None
58 -
59 -
60 -class SigmaQueryOutResponse(BaseModel):
61 - """
62 - Represents the Sigma query output response.
63 - """
64 -
65 - sigma_queries: Optional[List[SigmaQueriesOut]] = []
66 - success: bool
67 - message: str
68 -
69 -
70 -class ActivateSigmaQueryResponse(BaseModel):
71 - """
72 - Represents the Sigma query activation response.
73 - """
74 -
75 - success: bool
76 - message: str
77 - enabled_queries: List[str] = []
78 -
79 -
80 -class DeactivateSigmaQueryResponse(BaseModel):
81 - """
82 - Represents the Sigma query deactivation response.
83 - """
84 -
85 - success: bool
86 - message: str
87 - disabled_queries: List[str] = []
88 -
89 -
90 -class DeleteSigmaQueryResponse(BaseModel):
91 - """
92 - Represents the Sigma query deletion response.
93 - """
94 -
95 - success: bool
96 - message: str
97 - deleted_queries: List[str] = []
98 -
99 -
100 -class RunActiveSigmaQueries(BaseModel):
101 - query: str = Field(
102 - ...,
103 - description="The query to run.",
104 - )
105 - time_interval: str = Field(
106 - ...,
107 - description="The time interval to run the query for.",
108 - )
109 - last_execution_time: datetime = None
110 - rule_name: str = Field(
111 - ...,
112 - description="The name of the rule.",
113 - )
114 - index: str = Field(
115 - "wazuh*",
116 - description="The index to run the query on.",
117 - )
118 -
119 -
120 -class CreateSigmaQuery(BaseModel):
121 - """
122 - Represents the Sigma query creation request.
123 - """
124 -
125 - rule_name: str
126 - rule_query: str
127 - active: bool
128 - time_interval: str
129 -
130 - @validator("rule_name")
131 - def validate_rule_name(cls, rule_name: str) -> str:
132 - """
133 - Validates the rule name.
134 -
135 - Args:
136 - rule_name (str): The rule name.
137 -
138 - Returns:
139 - str: The validated rule name.
140 - """
141 - if len(rule_name) < 1:
142 - raise ValueError("The rule name must not be empty.")
143 -
144 - return rule_name
145 -
146 - @validator("rule_query")
147 - def validate_rule_query(cls, rule_query: str) -> str:
148 - """
149 - Validates the rule query.
150 -
151 - Args:
152 - rule_query (str): The rule query.
153 -
154 - Returns:
155 - str: The validated rule query.
156 - """
157 - if len(rule_query) < 1:
158 - raise ValueError("The rule query must not be empty.")
159 -
160 - return rule_query
161 -
162 - @validator("time_interval")
163 - def validate_time_interval(cls, time_interval: str) -> str:
164 - """
165 - Validates the time interval.
166 -
167 - Args:
168 - time_interval (str): The time interval.
169 -
170 - Returns:
171 - str: The validated time interval.
172 - """
173 - if not re.match(r"^\d+[mhd]$", time_interval):
174 - raise ValueError("The time interval must be in the format of minutes (e.g., '1m'), hours (e.g., '1h'), or days (e.g., '1d').")
175 -
176 - return time_interval
177 -
178 -
179 -class QueryString(BaseModel):
180 - query: str
181 - analyze_wildcard: bool
182 -
183 -
184 -class MustItem(BaseModel):
185 - query_string: QueryString
186 -
187 -
188 -class BoolQuery(BaseModel):
189 - must: List[MustItem]
190 -
191 -
192 -class Query(BaseModel):
193 - bool: BoolQuery
194 -
195 -
196 -class SigmaQueryGenerationResponse(BaseModel):
197 - query: Query
198 -
199 -
200 -class UpdateSigmaActive(BaseModel):
201 - rule_name: str
202 - active: bool
203 -
204 -
205 -class UpdateSigmaTimeInterval(BaseModel):
206 - rule_name: str
207 - time_interval: str
208 -
209 - @validator("time_interval")
210 - def validate_time_interval(cls, time_interval: str) -> str:
211 - """
212 - Validates the time interval.
213 -
214 - Args:
215 - time_interval (str): The time interval.
216 -
217 - Returns:
218 - str: The validated time interval.
219 - """
220 - if not re.match(r"^\d+[mhd]$", time_interval):
221 - raise ValueError("The time interval must be in the format of minutes (e.g., '1m'), hours (e.g., '1h'), or days (e.g., '1d').")
222 -
223 - return time_interval
backend/app/connectors/wazuh_indexer/services/sigma/elasticsearch.py deleted
-458
@@ -1,458 +0,0 @@
1 -import json
2 -import re
3 -from typing import Any
4 -from typing import ClassVar
5 -from typing import Dict
6 -from typing import Iterable
7 -from typing import List
8 -from typing import Optional
9 -from typing import Pattern
10 -from typing import Tuple
11 -from typing import Union
12 -
13 -import sigma
14 -from sigma.conditions import ConditionAND
15 -from sigma.conditions import ConditionFieldEqualsValueExpression
16 -from sigma.conditions import ConditionItem
17 -from sigma.conditions import ConditionNOT
18 -from sigma.conditions import ConditionOR
19 -from sigma.conversion.base import TextQueryBackend
20 -from sigma.conversion.deferred import DeferredQueryExpression
21 -from sigma.conversion.state import ConversionState
22 -from sigma.data.mitre_attack import mitre_attack_tactics
23 -from sigma.data.mitre_attack import mitre_attack_techniques
24 -from sigma.rule import SigmaRule
25 -from sigma.rule import SigmaRuleTag
26 -from sigma.types import SigmaCompareExpression
27 -from sigma.types import SigmaNull
28 -
29 -
30 -class LuceneBackend(TextQueryBackend):
31 - """
32 - Elasticsearch query string backend. Generates query strings described here in the
33 - Elasticsearch documentation:
34 -
35 - https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax
36 - """
37 -
38 - # A descriptive name of the backend
39 - name: ClassVar[str] = "Elasticsearch Lucene"
40 - # Output formats provided by the backend as name -> description mapping.
41 - # The name should match to finalize_output_<name>.
42 - formats: ClassVar[Dict[str, str]] = {
43 - "default": "Plain Elasticsearch Lucene queries",
44 - "kibana_ndjson": "Kibana NDJSON import file with Lucene queries",
45 - "dsl_lucene": "Elasticsearch query DSL with embedded Lucene queries",
46 - "siem_rule": "Elasticsearch query DSL as SIEM Rules in JSON Format",
47 - "siem_rule_ndjson": "Elasticsearch query DSL as SIEM Rules in NDJSON Format",
48 - }
49 - # Does the backend requires that a processing pipeline is provided?
50 - requires_pipeline: ClassVar[bool] = True
51 -
52 - # Operator precedence: tuple of Condition{AND,OR,NOT} in order of precedence.
53 - # The backend generates grouping if required
54 - precedence: ClassVar[Tuple[ConditionItem, ConditionItem, ConditionItem]] = (
55 - ConditionNOT,
56 - ConditionOR,
57 - ConditionAND,
58 - )
59 - # Expression for precedence override grouping as format string with {expr} placeholder
60 - group_expression: ClassVar[str] = "({expr})"
61 - parenthesize: bool = True
62 -
63 - # Generated query tokens
64 - token_separator: str = " " # separator inserted between all boolean operators
65 - or_token: ClassVar[str] = "OR"
66 - and_token: ClassVar[str] = "AND"
67 - not_token: ClassVar[str] = "NOT"
68 - # Token inserted between field and value (without separator)
69 - eq_token: ClassVar[str] = ":"
70 -
71 - # String output
72 - # Fields
73 - # No quoting of field names
74 - # Escaping
75 - # Character to escape particular parts defined in field_escape_pattern.
76 - field_escape: ClassVar[str] = "\\"
77 - # All matches of this pattern are prepended with the string contained in field_escape.
78 - field_escape_pattern: ClassVar[Pattern] = re.compile("[\\s*]")
79 -
80 - # Values
81 - # string quoting character (added as escaping character)
82 - str_quote: ClassVar[str] = '"'
83 - str_quote_pattern: ClassVar[Pattern] = re.compile(r"^$")
84 - str_quote_pattern_negation: ClassVar[bool] = False
85 - # Escaping character for special characrers inside string
86 - escape_char: ClassVar[str] = "\\"
87 - # Character used as multi-character wildcard
88 - wildcard_multi: ClassVar[str] = "*"
89 - # Character used as single-character wildcard
90 - wildcard_single: ClassVar[str] = "?"
91 - # Characters quoted in addition to wildcards and string quote
92 - add_escaped: ClassVar[str] = '+-=&|!(){}[]<>^"~*?:\\/ '
93 - bool_values: ClassVar[Dict[bool, str]] = { # Values to which boolean values are mapped.
94 - True: "true",
95 - False: "false",
96 - }
97 -
98 - # Regular expressions
99 - # Regular expression query as format string with placeholders {field} and {regex}
100 - re_expression: ClassVar[str] = "{field}:/{regex}/"
101 - # Character used for escaping in regular expressions
102 - re_escape_char: ClassVar[str] = "\\"
103 - re_escape: ClassVar[Tuple[str]] = ("/",)
104 - # Don't escape the escape char
105 - re_escape_escape_char: ClassVar[bool] = False
106 -
107 - # cidr expressions
108 - # CIDR expression query as format string with placeholders {field} = {value}
109 - cidr_expression: ClassVar[str] = "{field}:{network}\\/{prefixlen}"
110 -
111 - # Numeric comparison operators
112 - # Compare operation query as format string with placeholders {field}, {operator} and {value}
113 - compare_op_expression: ClassVar[str] = "{field}:{operator}{value}"
114 - # Mapping between CompareOperators elements and strings used as replacement
115 - # for {operator} in compare_op_expression
116 - compare_operators: ClassVar[Dict[SigmaCompareExpression.CompareOperators, str]] = {
117 - SigmaCompareExpression.CompareOperators.LT: "<",
118 - SigmaCompareExpression.CompareOperators.LTE: "<=",
119 - SigmaCompareExpression.CompareOperators.GT: ">",
120 - SigmaCompareExpression.CompareOperators.GTE: ">=",
121 - }
122 -
123 - # Null/None expressions
124 - # Expression for field has null value as format string with {field} placeholder for field name
125 - field_null_expression: ClassVar[str] = "NOT _exists_:{field}"
126 -
127 - # Field value in list, e.g. "field in (value list)" or "field containsall (value list)"
128 - # Convert OR as in-expression
129 - convert_or_as_in: ClassVar[bool] = True
130 - # Convert AND as in-expression
131 - convert_and_as_in: ClassVar[bool] = False
132 - # Values in list can contain wildcards. If set to False (default)
133 - # only plain values are converted into in-expressions.
134 - in_expressions_allow_wildcards: ClassVar[bool] = True
135 - # Expression for field in list of values as format string with
136 - # placeholders {field}, {op} and {list}
137 - field_in_list_expression: ClassVar[str] = "{field}{op}({list})"
138 - # Operator used to convert OR into in-expressions. Must be set if convert_or_as_in is set
139 - or_in_operator: ClassVar[str] = ":"
140 - # List element separator
141 - list_separator: ClassVar[str] = " OR "
142 -
143 - # Value not bound to a field
144 - # Expression for string value not bound to a field as format string with placeholder {value}
145 - unbound_value_str_expression: ClassVar[str] = "*{value}*"
146 - # Expression for number value not bound to a field as format string with placeholder {value}
147 - unbound_value_num_expression: ClassVar[str] = "{value}"
148 -
149 - def __init__(
150 - self,
151 - processing_pipeline: Optional["sigma.processing.pipeline.ProcessingPipeline"] = None,
152 - collect_errors: bool = False,
153 - index_names: List = [
154 - "apm-*-transaction*",
155 - "auditbeat-*",
156 - "endgame-*",
157 - "filebeat-*",
158 - "logs-*",
159 - "packetbeat-*",
160 - "traces-apm*",
161 - "winlogbeat-*",
162 - "-*elastic-cloud-logs-*",
163 - ],
164 - schedule_interval: int = 5,
165 - schedule_interval_unit: str = "m",
166 - **kwargs,
167 - ):
168 - super().__init__(processing_pipeline, collect_errors, **kwargs)
169 - self.index_names = index_names or [
170 - "apm-*-transaction*",
171 - "auditbeat-*",
172 - "endgame-*",
173 - "filebeat-*",
174 - "logs-*",
175 - "packetbeat-*",
176 - "traces-apm*",
177 - "winlogbeat-*",
178 - "-*elastic-cloud-logs-*",
179 - ]
180 - self.schedule_interval = schedule_interval or 5
181 - self.schedule_interval_unit = schedule_interval_unit or "m"
182 - self.severity_risk_mapping = {
183 - "INFORMATIONAL": 1,
184 - "LOW": 21,
185 - "MEDIUM": 47,
186 - "HIGH": 73,
187 - "CRITICAL": 99,
188 - }
189 -
190 - @staticmethod
191 - def _is_field_null_condition(cond: ConditionItem) -> bool:
192 - return isinstance(cond, ConditionFieldEqualsValueExpression) and isinstance(cond.value, SigmaNull)
193 -
194 - def convert_condition_not(self, cond: ConditionNOT, state: ConversionState) -> Union[str, DeferredQueryExpression]:
195 - """When checking if a field is not null, convert "NOT NOT _exists_:field" to "_exists_:field"."""
196 - if LuceneBackend._is_field_null_condition(cond.args[0]):
197 - return f"_exists_:{cond.args[0].field}"
198 -
199 - return super().convert_condition_not(cond, state)
200 -
201 - def convert_condition_field_eq_val_cidr(
202 - self,
203 - cond: ConditionFieldEqualsValueExpression,
204 - state: ConversionState,
205 - ) -> Union[str, DeferredQueryExpression]:
206 - if ":" in cond.value.cidr:
207 - return super().convert_condition_field_eq_val_cidr(cond, state).replace(":", r"\:").replace(r"\:", ":", 1)
208 - else:
209 - return super().convert_condition_field_eq_val_cidr(cond, state)
210 -
211 - def convert_condition_field_eq_expansion(self, cond: ConditionFieldEqualsValueExpression, state: ConversionState) -> Any:
212 - """
213 - Convert each value of the expansion with the field from the containing condition and OR-link
214 - all converted subconditions.
215 - """
216 - or_cond = ConditionOR(
217 - [ConditionFieldEqualsValueExpression(cond.field, value) for value in cond.value.values],
218 - cond.source,
219 - )
220 - if self.decide_convert_condition_as_in_expression(or_cond, state):
221 - return self.convert_condition_as_in_expression(or_cond, state)
222 - else:
223 - return self.convert_condition_or(cond, state)
224 -
225 - def compare_precedence(self, outer: ConditionItem, inner: ConditionItem) -> bool:
226 - """Override precedence check for null field conditions."""
227 - if isinstance(inner, ConditionNOT) and LuceneBackend._is_field_null_condition(inner.args[0]):
228 - # inner will turn into "_exists_:field", no parentheses needed
229 - return True
230 -
231 - if LuceneBackend._is_field_null_condition(inner):
232 - # inner will turn into "NOT _exists_:field", force parentheses
233 - return False
234 -
235 - return super().compare_precedence(outer, inner)
236 -
237 - def finalize_output_threat_model(self, tags: List[SigmaRuleTag]) -> Iterable[Dict]:
238 - attack_tags = [t for t in tags if t.namespace == "attack"]
239 - if not len(attack_tags) >= 2:
240 - return []
241 -
242 - techniques = [tag.name.upper() for tag in attack_tags if re.match(r"[tT]\d{4}", tag.name)]
243 - tactics = [tag.name.lower() for tag in attack_tags if not re.match(r"[tT]\d{4}", tag.name)]
244 -
245 - for tactic, technique in zip(tactics, techniques):
246 - if not tactic or not technique: # Only add threat if tactic and technique is known
247 - continue
248 -
249 - try:
250 - if "." in technique: # Contains reference to Mitre Att&ck subtechnique
251 - sub_technique = technique
252 - technique = technique[0:5]
253 - sub_technique_name = mitre_attack_techniques[sub_technique]
254 -
255 - sub_techniques = [
256 - {
257 - "id": sub_technique,
258 - "reference": f"https://attack.mitre.org/techniques/{sub_technique.replace('.', '/')}",
259 - "name": sub_technique_name,
260 - },
261 - ]
262 - else:
263 - sub_techniques = []
264 -
265 - tactic_id = [id for (id, name) in mitre_attack_tactics.items() if name == tactic.replace("_", "-")][0]
266 - technique_name = mitre_attack_techniques[technique]
267 - except (IndexError, KeyError):
268 - # Occurs when Sigma Mitre Att&ck list is out of date
269 - continue
270 -
271 - yield {
272 - "tactic": {
273 - "id": tactic_id,
274 - "reference": f"https://attack.mitre.org/tactics/{tactic_id}",
275 - "name": tactic.title().replace("_", " "),
276 - },
277 - "framework": "MITRE ATT&CK",
278 - "technique": [
279 - {
280 - "id": technique,
281 - "reference": f"https://attack.mitre.org/techniques/{technique}",
282 - "name": technique_name,
283 - "subtechnique": sub_techniques,
284 - },
285 - ],
286 - }
287 -
288 - for tag in attack_tags:
289 - tags.remove(tag)
290 -
291 - def finalize_query_dsl_lucene(self, rule: SigmaRule, query: str, index: int, state: ConversionState) -> Dict:
292 - return {"query": {"bool": {"must": [{"query_string": {"query": query, "analyze_wildcard": True}}]}}}
293 -
294 - def finalize_output_dsl_lucene(self, queries: List[Dict]) -> Dict:
295 - return list(queries)
296 -
297 - def finalize_query_kibana_ndjson(self, rule: SigmaRule, query: str, index: int, state: ConversionState) -> Dict:
298 - # TODO: implement the per-query output for the output format kibana here. Usually, the
299 - # generated query is embedded into a template, e.g. a JSON format with additional
300 - # information from the Sigma rule.
301 - columns = []
302 - index = "beats-*"
303 - ndjson = {
304 - "id": str(rule.id),
305 - "type": "search",
306 - "attributes": {
307 - "title": f"SIGMA - {rule.title}",
308 - "description": rule.description,
309 - "hits": 0,
310 - "columns": columns,
311 - "sort": ["@timestamp", "desc"],
312 - "version": 1,
313 - "kibanaSavedObjectMeta": {
314 - "searchSourceJSON": str(
315 - json.dumps(
316 - {
317 - "index": index,
318 - "filter": [],
319 - "highlight": {
320 - "pre_tags": ["@kibana-highlighted-field@"],
321 - "post_tags": ["@/kibana-highlighted-field@"],
322 - "fields": {"*": {}},
323 - "require_field_match": False,
324 - "fragment_size": 2147483647,
325 - },
326 - "query": {
327 - "query_string": {
328 - "query": query,
329 - "analyze_wildcard": True,
330 - },
331 - },
332 - },
333 - ),
334 - ),
335 - },
336 - },
337 - "references": [
338 - {
339 - "id": index,
340 - "name": "kibanaSavedObjectMeta.searchSourceJSON.index",
341 - "type": "index-pattern",
342 - },
343 - ],
344 - }
345 - return ndjson
346 -
347 - def finalize_output_kibana_ndjson(self, queries: List[str]) -> List[Dict]:
348 - # TODO: implement the output finalization for all generated queries for the format kibana
349 - # here. Usually, the single generated queries are embedded into a structure, e.g. some
350 - # JSON or XML that can be imported into the SIEM.
351 - return list(queries)
352 -
353 - def finalize_query_siem_rule(self, rule: SigmaRule, query: str, index: int, state: ConversionState) -> Dict:
354 - """
355 - Create SIEM Rules in JSON Format. These rules could be imported into Kibana using the
356 - Create Rule API https://www.elastic.co/guide/en/kibana/8.6/create-rule-api.html
357 - This API (and generated data) is NOT the same like importing Detection Rules via:
358 - Kibana -> Security -> Alerts -> Manage Rules -> Import
359 - If you want to have a nice importable NDJSON File for the Security Rule importer
360 - use pySigma Format 'siem_rule_ndjson' instead.
361 - """
362 -
363 - siem_rule = {
364 - "name": f"SIGMA - {rule.title}",
365 - "consumer": "siem",
366 - "enabled": True,
367 - "throttle": None,
368 - "schedule": {"interval": f"{self.schedule_interval}{self.schedule_interval_unit}"},
369 - "params": {
370 - "author": [rule.author] if rule.author is not None else [],
371 - "description": (rule.description if rule.description is not None else "No description"),
372 - "ruleId": str(rule.id),
373 - "falsePositives": rule.falsepositives,
374 - "from": f"now-{self.schedule_interval}{self.schedule_interval_unit}",
375 - "immutable": False,
376 - "license": "DRL",
377 - "outputIndex": "",
378 - "meta": {
379 - "from": "1m",
380 - },
381 - "maxSignals": 100,
382 - "riskScore": (self.severity_risk_mapping[rule.level.name] if rule.level is not None else 21),
383 - "riskScoreMapping": [],
384 - "severity": (str(rule.level.name).lower() if rule.level is not None else "low"),
385 - "severityMapping": [],
386 - "threat": list(self.finalize_output_threat_model(rule.tags)),
387 - "to": "now",
388 - "references": rule.references,
389 - "version": 1,
390 - "exceptionsList": [],
391 - "relatedIntegrations": [],
392 - "requiredFields": [],
393 - "setup": "",
394 - "type": "query",
395 - "language": "lucene",
396 - "index": self.index_names,
397 - "query": query,
398 - "filters": [],
399 - },
400 - "rule_type_id": "siem.queryRule",
401 - "tags": [f"{n.namespace}-{n.name}" for n in rule.tags],
402 - "notify_when": "onActiveAlert",
403 - "actions": [],
404 - }
405 - return siem_rule
406 -
407 - def finalize_output_siem_rule(self, queries: List[Dict]) -> Dict:
408 - return list(queries)
409 -
410 - def finalize_query_siem_rule_ndjson(self, rule: SigmaRule, query: str, index: int, state: ConversionState) -> Dict:
411 - """
412 - Generating SIEM/Detection Rules in NDJSON Format. Compatible with
413 -
414 - https://www.elastic.co/guide/en/security/8.6/rules-ui-management.html#import-export-rules-ui
415 - """
416 -
417 - siem_rule = {
418 - "id": str(rule.id),
419 - "name": f"SIGMA - {rule.title}",
420 - "enabled": True,
421 - "throttle": "no_actions",
422 - "interval": f"{self.schedule_interval}{self.schedule_interval_unit}",
423 - "author": [rule.author] if rule.author is not None else [],
424 - "description": (rule.description if rule.description is not None else "No description"),
425 - "rule_id": str(rule.id),
426 - "false_positives": rule.falsepositives,
427 - "from": f"now-{self.schedule_interval}{self.schedule_interval_unit}",
428 - "immutable": False,
429 - "license": "DRL",
430 - "output_index": "",
431 - "meta": {
432 - "from": "1m",
433 - },
434 - "max_signals": 100,
435 - "risk_score": (self.severity_risk_mapping[rule.level.name] if rule.level is not None else 21),
436 - "risk_score_mapping": [],
437 - "severity": (str(rule.level.name).lower() if rule.level is not None else "low"),
438 - "severity_mapping": [],
439 - "threat": list(self.finalize_output_threat_model(rule.tags)),
440 - "tags": [f"{n.namespace}-{n.name}" for n in rule.tags],
441 - "to": "now",
442 - "references": rule.references,
443 - "version": 1,
444 - "exceptions_list": [],
445 - "related_integrations": [],
446 - "required_fields": [],
447 - "setup": "",
448 - "type": "query",
449 - "language": "lucene",
450 - "index": self.index_names,
451 - "query": query,
452 - "filters": [],
453 - "actions": [],
454 - }
455 - return siem_rule
456 -
457 - def finalize_output_siem_rule_ndjson(self, queries: List[Dict]) -> Dict:
458 - return list(queries)
backend/app/connectors/wazuh_indexer/services/sigma/execute_query.py deleted
-158
@@ -1,158 +0,0 @@
1 -from datetime import datetime
2 -from typing import List
3 -
4 -from fastapi import HTTPException
5 -from loguru import logger
6 -from sqlalchemy.ext.asyncio import AsyncSession
7 -
8 -from app.connectors.wazuh_indexer.schema.sigma import RunActiveSigmaQueries
9 -from app.connectors.wazuh_indexer.utils.universal import (
10 - create_wazuh_indexer_client_async,
11 -)
12 -from app.incidents.schema.incident_alert import CreatedAlertPayload
13 -from app.incidents.services.incident_alert import add_asset_to_copilot_alert
14 -from app.incidents.services.incident_alert import build_alert_context_payload
15 -from app.incidents.services.incident_alert import create_alert_full
16 -from app.incidents.services.incident_alert import get_all_field_names
17 -from app.incidents.services.incident_alert import get_customer_code
18 -from app.incidents.services.incident_alert import is_customer_code_valid
19 -from app.incidents.services.incident_alert import open_alert_exists
20 -
21 -
22 -async def build_alert_payload(
23 - sigma_rule_name: str,
24 - syslog_type: str,
25 - index_name: str,
26 - index_id: str,
27 - alert_payload: dict,
28 - session: AsyncSession,
29 -) -> CreatedAlertPayload:
30 - field_names = await get_all_field_names(syslog_type, session)
31 - validate_field_names(field_names, alert_payload)
32 - return await create_alert_payload(sigma_rule_name, syslog_type, index_name, index_id, alert_payload, field_names)
33 -
34 -
35 -def validate_field_names(field_names, alert_payload):
36 - for field_name in [field_names.asset_name, field_names.timefield_name, field_names.alert_title_name]:
37 - if field_name not in alert_payload:
38 - raise HTTPException(
39 - status_code=400,
40 - detail=f"Field name {field_name} not found in alert payload",
41 - )
42 -
43 -
44 -async def create_alert_payload(sigma_rule_name, syslog_type, index_name, index_id, alert_payload, field_names):
45 - return CreatedAlertPayload(
46 - alert_context_payload=await build_alert_context_payload(alert_payload, field_names),
47 - asset_payload=alert_payload.get(field_names.asset_name),
48 - timefield_payload=alert_payload.get(field_names.timefield_name),
49 - alert_title_payload="SIGMA Alert: " + sigma_rule_name,
50 - source=syslog_type,
51 - index_name=index_name,
52 - index_id=index_id,
53 - )
54 -
55 -
56 -async def format_opensearch_query(query: str, time_interval: str, last_execution_time: datetime) -> dict:
57 - logger.info(f"Last execution time: {last_execution_time}")
58 - formatted_last_execution_time = last_execution_time.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
59 - return {
60 - "query": {
61 - "bool": {
62 - "must": [
63 - {
64 - "query_string": {
65 - "query": query,
66 - "fields": [],
67 - "type": "best_fields",
68 - "default_operator": "or",
69 - "max_determinized_states": 10000,
70 - "enable_position_increments": True,
71 - "fuzziness": "AUTO",
72 - "fuzzy_prefix_length": 0,
73 - "fuzzy_max_expansions": 50,
74 - "phrase_slop": 0,
75 - "analyze_wildcard": True,
76 - "escape": False,
77 - "auto_generate_synonyms_phrase_query": True,
78 - "fuzzy_transpositions": True,
79 - "boost": 1,
80 - },
81 - },
82 - {
83 - "range": {
84 - "timestamp": {
85 - # "from": f"now-{time_interval}",
86 - "from": formatted_last_execution_time,
87 - "to": "now",
88 - "include_lower": True,
89 - "include_upper": False,
90 - "boost": 1,
91 - },
92 - },
93 - },
94 - ],
95 - "adjust_pure_negative": True,
96 - "boost": 1,
97 - },
98 - },
99 - }
100 -
101 -
102 -async def send_query_to_opensearch(
103 - es_client,
104 - query: dict,
105 - rule_name: str,
106 - index: str = "wazuh*",
107 - session: AsyncSession = None,
108 -) -> List[dict]:
109 - try:
110 - response = await es_client.search(index=index, body=query)
111 - logger.info(f"Response: {response}")
112 - hits = response["hits"]["hits"]
113 - return await process_hits(hits, rule_name, session)
114 - except Exception as e:
115 - logger.error(f"Error executing query: {e}")
116 - return []
117 -
118 -
119 -async def process_hits(hits, rule_name, session: AsyncSession):
120 - logger.info(f"Processing number of hits: {len(hits)}")
121 - results = []
122 - for hit in hits:
123 - doc_id = hit["_id"]
124 - index = hit["_index"]
125 - customer_code = await get_customer_code(alert_details=hit["_source"])
126 - await is_customer_code_valid(customer_code=customer_code, session=session)
127 - alert_payload = await build_alert_payload(
128 - sigma_rule_name=rule_name,
129 - syslog_type="wazuh",
130 - index_name=index,
131 - index_id=doc_id,
132 - alert_payload=hit["_source"],
133 - session=session,
134 - )
135 - logger.info(f"Alert payload: {alert_payload}")
136 - existing_alert = await open_alert_exists(alert_payload, customer_code, session)
137 - if existing_alert:
138 - logger.info(f"Alert already exists: {existing_alert}")
139 - await add_asset_to_copilot_alert(
140 - alert_payload=alert_payload,
141 - alert_id=existing_alert,
142 - customer_code=customer_code,
143 - session=session,
144 - )
145 - results.append(existing_alert)
146 - else:
147 - new_alert = await create_alert_full(alert_payload=alert_payload, customer_code=customer_code, session=session)
148 - results.append(new_alert)
149 - return results
150 -
151 -
152 -async def execute_query(payload: RunActiveSigmaQueries, session: AsyncSession = None):
153 - client = await create_wazuh_indexer_client_async()
154 - formatted_query = await format_opensearch_query(payload.query, payload.time_interval, payload.last_execution_time)
155 - logger.info(f"Executing query: {formatted_query}")
156 - results = await send_query_to_opensearch(client, formatted_query, payload.rule_name, index=payload.index, session=session)
157 - logger.info(f"Results: {results}")
158 - return results
backend/app/connectors/wazuh_indexer/services/sigma/generate_query.py deleted
-41
@@ -1,41 +0,0 @@
1 -from sigma.collection import SigmaCollection
2 -from sigma.processing.resolver import ProcessingPipelineResolver
3 -
4 -from app.connectors.wazuh_indexer.schema.sigma import SigmaQueryGenerationResponse
5 -from app.connectors.wazuh_indexer.services.sigma.opensearch import (
6 - OpensearchLuceneBackend,
7 -)
8 -from app.connectors.wazuh_indexer.services.sigma.sysmon import sysmon_pipeline
9 -from app.connectors.wazuh_indexer.services.sigma.windows import ecs_windows
10 -
11 -
12 -async def create_sigma_query_from_rule(rule: str) -> SigmaQueryGenerationResponse:
13 - # Create our pipeline resolver
14 - piperesolver = ProcessingPipelineResolver()
15 -
16 - # Add wanted pipelines
17 - piperesolver.add_pipeline_class(ecs_windows())
18 - piperesolver.add_pipeline_class(sysmon_pipeline())
19 -
20 - # Create a single sorted and prioritzed pipeline
21 - resolved_pipeline = piperesolver.resolve(piperesolver.pipelines)
22 -
23 - # Instantiate backend, using our resolved pipeline
24 - # and some backend parameter
25 - backend = OpensearchLuceneBackend(
26 - resolved_pipeline,
27 - index_names=["logs-*-*", "beats-*"],
28 - monitor_interval=10,
29 - monitor_interval_unit="MINUTES",
30 - )
31 -
32 - rules = SigmaCollection.from_yaml(rule)
33 -
34 - # Convert the rule to DSL format
35 - dsl_result = backend.convert(rules, output_format="dsl_lucene")[0]
36 -
37 - # Create the Pydantic model from the DSL result
38 - sigma_query_out_response = SigmaQueryGenerationResponse.parse_obj(dsl_result)
39 -
40 - # return the DSL query as a string
41 - return sigma_query_out_response
backend/app/connectors/wazuh_indexer/services/sigma/opensearch.py deleted
-82
@@ -1,82 +0,0 @@
1 -from typing import ClassVar
2 -from typing import Dict
3 -from typing import List
4 -from typing import Optional
5 -
6 -import sigma
7 -from sigma.conversion.state import ConversionState
8 -from sigma.rule import SigmaRule
9 -
10 -from app.connectors.wazuh_indexer.services.sigma.elasticsearch import LuceneBackend
11 -
12 -
13 -class OpensearchLuceneBackend(LuceneBackend):
14 - """OpensearchLuceneBackend backend."""
15 -
16 - name: ClassVar[str] = "OpenSearch Lucene" # A descriptive name of the backend
17 - formats: ClassVar[
18 - Dict[str, str]
19 - ] = { # Output formats provided by the backend as name -> description mapping. The name should match to finalize_output_<name>.
20 - "default": "Plain OpenSearch Lucene queries",
21 - "dashboards_ndjson": "OpenSearch Dashboards NDJSON import file with Lucene queries",
22 - "monitor_rule": "OpenSearch monitor rule with embedded Lucene query",
23 - "dsl_lucene": "OpenSearch query DSL with embedded Lucene queries",
24 - }
25 - # Does the backend requires that a processing pipeline is provided?
26 - requires_pipeline: ClassVar[bool] = True
27 -
28 - def __init__(
29 - self,
30 - processing_pipeline: Optional["sigma.processing.pipeline.ProcessingPipeline"] = None,
31 - collect_errors: bool = False,
32 - index_names: List = ["beats-*"],
33 - monitor_interval: int = 5,
34 - monitor_interval_unit: str = "MINUTES",
35 - **kwargs,
36 - ):
37 - super().__init__(processing_pipeline, collect_errors, **kwargs)
38 - self.index_names = index_names or ["beats-*"]
39 - self.monitor_interval = monitor_interval or 5
40 - self.monitor_interval_unit = monitor_interval_unit or "MINUTES"
41 -
42 - def finalize_query_monitor_rule(self, rule: SigmaRule, query: str, index: int, state: ConversionState) -> dict:
43 - severity_mapping = {5: 1, 4: 2, 3: 3, 2: 4, 1: 5}
44 - monitor_rule = {
45 - "type": "monitor",
46 - "name": f"SIGMA - {rule.title}",
47 - "description": rule.description,
48 - "enabled": True,
49 - "schedule": {"period": {"interval": self.monitor_interval, "unit": self.monitor_interval_unit}},
50 - "inputs": [
51 - {
52 - "search": {
53 - "indices": self.index_names,
54 - "query": {"size": 1, "query": {"bool": {"must": [{"query_string": {"query": query, "analyze_wildcard": True}}]}}},
55 - },
56 - },
57 - ],
58 - "tags": [f"{n.namespace}-{n.name}" for n in rule.tags],
59 - "triggers": [
60 - {
61 - "name": "generated-trigger",
62 - "severity": severity_mapping[rule.level.value] if rule.level is not None else 1,
63 - "condition": {"script": {"source": "ctx.results[0].hits.total.value > 0", "lang": "painless"}},
64 - "actions": [],
65 - },
66 - ],
67 - "sigma_meta_data": {"rule_id": str(rule.id), "threat": []},
68 - "references": rule.references,
69 - }
70 -
71 - return monitor_rule
72 -
73 - def finalize_output_monitor_rule(self, queries: List[str]) -> str:
74 - return list(queries)
75 -
76 - def finalize_query_dashboards_ndjson(self, rule: SigmaRule, query: str, index: int, state: ConversionState) -> str:
77 - """Alias to Kibana NDJSON query finalization."""
78 - return self.finalize_query_kibana_ndjson(rule, query, index, state)
79 -
80 - def finalize_output_dashboards_ndjson(self, queries: List[str]) -> str:
81 - """Alias to Kibana NDJSON output finalization."""
82 - return self.finalize_output_kibana_ndjson(queries)
backend/app/connectors/wazuh_indexer/services/sigma/sigma_db_operations.py deleted
-316
@@ -1,316 +0,0 @@
1 -import os
2 -import re
3 -from datetime import datetime
4 -from datetime import timedelta
5 -from typing import List
6 -
7 -import yaml
8 -from fastapi import HTTPException
9 -from loguru import logger
10 -from sqlalchemy.exc import IntegrityError
11 -from sqlalchemy.ext.asyncio import AsyncSession
12 -from sqlalchemy.future import select
13 -
14 -from app.connectors.wazuh_indexer.models.sigma import SigmaQuery
15 -from app.connectors.wazuh_indexer.schema.sigma import CreateSigmaQuery
16 -from app.connectors.wazuh_indexer.schema.sigma import SigmaRuleUploadRequest
17 -from app.connectors.wazuh_indexer.services.sigma.generate_query import (
18 - create_sigma_query_from_rule,
19 -)
20 -from app.connectors.wazuh_indexer.services.sigma.sigma_download import find_yaml_files
21 -
22 -
23 -def parse_time_interval(interval: str) -> timedelta:
24 - match = re.match(r"(\d+)([smhd])", interval)
25 - if not match:
26 - raise ValueError(f"Invalid time interval format: {interval}")
27 - value, unit = match.groups()
28 - value = int(value)
29 - if unit == "s":
30 - return timedelta(seconds=value)
31 - elif unit == "m":
32 - return timedelta(minutes=value)
33 - elif unit == "h":
34 - return timedelta(hours=value)
35 - elif unit == "d":
36 - return timedelta(days=value)
37 - else:
38 - raise ValueError(f"Invalid time unit: {unit}")
39 -
40 -
41 -def check_level(rule_levels: list, file_path):
42 - delete_file = False
43 - with open(file_path, "r", encoding="utf-8") as file:
44 - for line in file:
45 - if line.startswith("level:"):
46 - level = line.split(":")[1].strip()
47 - if level not in rule_levels:
48 - delete_file = True
49 - break
50 - if delete_file:
51 - logger.info(f"Deleting file: {file_path}")
52 - os.remove(file_path) # delete the file if its level is not high or critical
53 - return False
54 - return True
55 -
56 -
57 -def validate_title(title):
58 - # Check if title is a string
59 - if not isinstance(title, str):
60 - return False
61 -
62 - # Check if title length is within the limit
63 - if len(title) > 65535:
64 - return False
65 -
66 - # Check if title contains only allowed characters
67 - pattern = re.compile(r"^[a-zA-Z0-9,.\-_/ ]*$")
68 - if not pattern.match(title):
69 - return False
70 -
71 - return True
72 -
73 -
74 -async def extract_title(file_path):
75 - # Load the YAML file
76 - with open(file_path, "r", encoding="utf-8") as file:
77 - data = yaml.safe_load(file)
78 -
79 - # Extract the title field
80 - title = data.get("title", "")
81 -
82 - # Validate the title
83 - if not validate_title(title):
84 - # Remove disallowed characters
85 - cleaned_title = re.sub(r"[^a-zA-Z0-9,.\-_/ ]", "", title)
86 -
87 - # Update the title field in the YAML data
88 - data["title"] = cleaned_title
89 -
90 - # Write the updated YAML data back to the file
91 - with open(file_path, "w", encoding="utf-8") as file:
92 - yaml.safe_dump(data, file)
93 -
94 - return cleaned_title
95 -
96 - return title
97 -
98 -
99 -async def get_sigma_query_by_id(
100 - db: AsyncSession,
101 - sigma_query_id: int,
102 -) -> SigmaQuery:
103 - """
104 - Retrieves a Sigma query by ID.
105 -
106 - Args:
107 - db (AsyncSession): The database session.
108 - sigma_query_id (int): The ID of the Sigma query.
109 -
110 - Returns:
111 - SigmaQuery: The Sigma query.
112 - """
113 - # Retrieve the Sigma query
114 - sigma_query = await db.execute(select(SigmaQuery).filter_by(id=sigma_query_id))
115 - sigma_query = sigma_query.scalars().first()
116 -
117 - if not sigma_query:
118 - raise HTTPException(
119 - status_code=404,
120 - detail="The Sigma query does not exist.",
121 - )
122 -
123 - return sigma_query
124 -
125 -
126 -async def list_sigma_queries(
127 - db: AsyncSession,
128 -) -> List[SigmaQuery]:
129 - """
130 - Retrieves a list of Sigma queries.
131 -
132 - Args:
133 - db (AsyncSession): The database session.
134 -
135 - Returns:
136 - List[SigmaQuery]: A list of Sigma queries.
137 - """
138 - # Retrieve the Sigma queries
139 - sigma_queries = await db.execute(select(SigmaQuery))
140 - sigma_queries = sigma_queries.scalars().all()
141 -
142 - return sigma_queries
143 -
144 -
145 -async def list_active_sigma_queries(
146 - db: AsyncSession,
147 -) -> List[SigmaQuery]:
148 - """
149 - Retrieves a list of active Sigma queries.
150 -
151 - Args:
152 - db (AsyncSession): The database session.
153 -
154 - Returns:
155 - List[SigmaQuery]: A list of active Sigma queries.
156 - """
157 - # Retrieve the active Sigma queries
158 - sigma_queries = await db.execute(select(SigmaQuery).filter_by(active=True))
159 - sigma_queries = sigma_queries.scalars().all()
160 -
161 - return sigma_queries
162 -
163 -
164 -async def list_inactive_sigma_queries(
165 - db: AsyncSession,
166 -) -> List[SigmaQuery]:
167 - """
168 - Retrieves a list of inactive Sigma queries.
169 -
170 - Args:
171 - db (AsyncSession): The database session.
172 -
173 - Returns:
174 - List[SigmaQuery]: A list of inactive Sigma queries.
175 - """
176 - # Retrieve the inactive Sigma queries
177 - sigma_queries = await db.execute(select(SigmaQuery).filter_by(active=False))
178 - sigma_queries = sigma_queries.scalars().all()
179 -
180 - return sigma_queries
181 -
182 -
183 -async def create_sigma_query(
184 - sigma_query: CreateSigmaQuery,
185 - db: AsyncSession,
186 -) -> SigmaQuery:
187 - """
188 - Creates a Sigma query.
189 -
190 - Args:
191 - sigma_query (CreateSigmaQuery): The Sigma query to create.
192 - db (AsyncSession): The database session.
193 -
194 - Returns:
195 - SigmaQuery: The created Sigma query.
196 - """
197 - # Create the Sigma query
198 - new_sigma_query = SigmaQuery(
199 - rule_name=sigma_query.rule_name,
200 - rule_query=sigma_query.rule_query,
201 - active=sigma_query.active,
202 - time_interval=sigma_query.time_interval,
203 - )
204 -
205 - # Add the Sigma query to the database
206 - db.add(new_sigma_query)
207 -
208 - try:
209 - await db.commit()
210 - except IntegrityError as e:
211 - logger.error(f"Failed to create the Sigma query: {e}")
212 - raise HTTPException(
213 - status_code=400,
214 - detail="Failed to create the Sigma query.",
215 - )
216 -
217 - return new_sigma_query
218 -
219 -
220 -async def read_sigma_rule(file: str) -> str:
221 - with open(file, "r", encoding="utf-8") as f:
222 - return f.read()
223 -
224 -
225 -async def get_existing_query(rule_name: str, db: AsyncSession):
226 - result = await db.execute(select(SigmaQuery).filter_by(rule_name=rule_name))
227 - return result.scalars().first()
228 -
229 -
230 -async def update_sigma_query(existing_query: SigmaQuery, new_query: CreateSigmaQuery, db: AsyncSession):
231 - existing_query.rule_query = new_query.rule_query
232 - existing_query.active = new_query.active
233 - existing_query.time_interval = new_query.time_interval
234 - await db.commit()
235 -
236 -
237 -async def process_sigma_file(file: str, db: AsyncSession):
238 - rule = await read_sigma_rule(file)
239 - title = await extract_title(file)
240 - query = await create_sigma_query_from_rule(rule)
241 -
242 - new_query = CreateSigmaQuery(
243 - rule_name=title,
244 - rule_query=query.query.bool.must[0].query_string.query,
245 - active=False,
246 - time_interval="5m",
247 - )
248 -
249 - existing_query = await get_existing_query(title, db)
250 - if existing_query:
251 - logger.info(f"Updating existing Sigma query: {title}")
252 - await update_sigma_query(existing_query, new_query, db)
253 - else:
254 - logger.info(f"Creating new Sigma query: {title}")
255 - await create_sigma_query(new_query, db)
256 -
257 -
258 -async def add_sigma_queries_to_db(request: SigmaRuleUploadRequest, db: AsyncSession):
259 - yaml_files = list(await find_yaml_files())
260 - # Only add the CRITICAL Severity SIGMA files
261 - sigma_files = [file for file in yaml_files if check_level(request.rule_levels, file)]
262 - for file in sigma_files:
263 - await process_sigma_file(file, db)
264 - return None
265 -
266 -
267 -async def delete_query_from_db(query, db: AsyncSession):
268 - try:
269 - await db.delete(query)
270 - await db.commit()
271 - except Exception as e:
272 - logger.error(f"Failed to delete the Sigma query: {e}")
273 - raise HTTPException(
274 - status_code=400,
275 - detail="Failed to delete the Sigma query.",
276 - )
277 -
278 -
279 -async def delete_sigma_rule(rule_name: str, db: AsyncSession):
280 - query = await get_existing_query(rule_name, db)
281 - if query:
282 - await delete_query_from_db(query, db)
283 - else:
284 - raise HTTPException(
285 - status_code=404,
286 - detail="The Sigma rule does not exist.",
287 - )
288 - return None
289 -
290 -
291 -async def set_sigma_query_active(rule_name: str, active: bool, db: AsyncSession):
292 - query = await get_existing_query(rule_name, db)
293 - if query:
294 - query.active = active
295 - query.last_updated = datetime.now()
296 - await db.commit()
297 - else:
298 - raise HTTPException(
299 - status_code=404,
300 - detail="The Sigma rule does not exist.",
301 - )
302 - return query
303 -
304 -
305 -async def update_sigma_time_interval(rule_name: str, time_interval: str, db: AsyncSession):
306 - query = await get_existing_query(rule_name, db)
307 - if query:
308 - query.time_interval = time_interval
309 - query.last_updated = datetime.now()
310 - await db.commit()
311 - else:
312 - raise HTTPException(
313 - status_code=404,
314 - detail="The Sigma rule does not exist.",
315 - )
316 - return query
backend/app/connectors/wazuh_indexer/services/sigma/sigma_download.py deleted
-93
@@ -1,93 +0,0 @@
1 -import os
2 -import shutil
3 -import zipfile
4 -from pathlib import Path
5 -from typing import List
6 -from urllib.parse import urlparse
7 -
8 -import requests
9 -from fastapi import HTTPException
10 -from loguru import logger
11 -
12 -ALLOWED_HOSTS = {"github.com", "raw.githubusercontent.com"}
13 -
14 -
15 -async def download_and_extract_zip(url: str) -> None:
16 - """
17 - Downloads a zipped folder from the given URL and extracts its contents into the specified directory.
18 -
19 - Args:
20 - url (str): The URL of the zipped folder.
21 - extract_to (str): The directory to extract the contents to. Defaults to the current directory.
22 - """
23 - # local_zip_path = os.path.join(extract_to, "sigma_all_rules.zip")
24 - directory = "app/connectors/wazuh_indexer/sigma_artifacts"
25 - full_path = os.path.abspath(directory)
26 -
27 - logger.info(f"Checking directory: {full_path}")
28 - local_zip_path = os.path.join(full_path, "sigma_all_rules.zip")
29 -
30 - # Ensure the directory exists
31 - os.makedirs(full_path, exist_ok=True)
32 -
33 - # Download the zipped folder
34 - parsed_url = urlparse(url)
35 - if parsed_url.hostname not in ALLOWED_HOSTS:
36 - raise HTTPException(status_code=400, detail="Only approved Sigma download hosts are allowed.")
37 -
38 - response = requests.get(url)
39 - response.raise_for_status() # Check if the request was successful
40 -
41 - # Save the zipped folder to a local file
42 - with open(local_zip_path, "wb") as file:
43 - file.write(response.content)
44 -
45 - # Extract the contents of the zipped folder
46 - with zipfile.ZipFile(local_zip_path, "r") as zip_ref:
47 - extract_root = Path(full_path).resolve()
48 - for member_name in zip_ref.namelist():
49 - dest = (Path(full_path) / member_name).resolve()
50 - if not str(dest).startswith(str(extract_root)):
51 - raise ValueError(f"Zip Slip attempt blocked: {member_name}")
52 - zip_ref.extractall(full_path)
53 -
54 - # Remove the downloaded zip file
55 - os.remove(local_zip_path)
56 -
57 -
58 -async def keep_only_folder_directory(folder: str, directory: str = "app/connectors/wazuh_indexer/sigma_artifacts/rules"):
59 - """
60 - Removes all directories except the Windows directory from the specified directory.
61 -
62 - Args:
63 - directory (str): The directory to clean up.
64 - """
65 - for item in os.listdir(directory):
66 - if item != folder:
67 - item_path = os.path.join(directory, item)
68 - if os.path.isdir(item_path):
69 - logger.info(f"Removing directory and its contents: {item_path}")
70 - shutil.rmtree(item_path)
71 -
72 -
73 -async def find_yaml_files(directory: str = "app/connectors/wazuh_indexer/sigma_artifacts/rules/windows") -> List[str]:
74 - """
75 - Finds all YAML files in the specified directory.
76 -
77 - Args:
78 - directory (str): The directory to search for YAML files.
79 -
80 - Returns:
81 - List[str]: A list of YAML file paths.
82 - """
83 - yaml_files = []
84 -
85 - for root, _, files in os.walk(directory):
86 - for file in files:
87 - if file.endswith(".yml"):
88 - # yaml_files.append(os.path.join(root, file))
89 - # logger.info(f"Found YAML file: {file}")
90 - full_path = os.path.join(root, file)
91 - yaml_files.append(full_path)
92 -
93 - return yaml_files
backend/app/connectors/wazuh_indexer/services/sigma/sysmon.py deleted
-65
@@ -1,65 +0,0 @@
1 -from sigma.pipelines.base import Pipeline
2 -from sigma.processing.conditions import LogsourceCondition
3 -from sigma.processing.pipeline import ProcessingItem
4 -from sigma.processing.pipeline import ProcessingPipeline
5 -from sigma.processing.transformations import AddConditionTransformation
6 -from sigma.processing.transformations import ChangeLogsourceTransformation
7 -
8 -sysmon_generic_logsource_eventid_mapping = { # map generic Sigma log sources to Sysmon event ids
9 - "process_creation": 1,
10 - "file_change": 2,
11 - "network_connection": 3,
12 - "process_termination": 5,
13 - "sysmon_status": [4, 16],
14 - "driver_load": 6,
15 - "image_load": 7,
16 - "create_remote_thread": 8,
17 - "raw_access_thread": 9,
18 - "process_access": 10,
19 - "file_event": 11,
20 - "registry_add": 12,
21 - "registry_delete": 12,
22 - "registry_set": 13,
23 - "registry_rename": 14,
24 - "registry_event": [12, 13, 14],
25 - "create_stream_hash": 15,
26 - "pipe_created": [17, 18],
27 - "wmi_event": [19, 20, 21],
28 - "dns_query": 22,
29 - "file_delete": [23, 26],
30 - "clipboard_capture": 24,
31 - "process_tampering": 25,
32 - "sysmon_error": 255,
33 -}
34 -
35 -
36 -@Pipeline
37 -def sysmon_pipeline() -> ProcessingPipeline:
38 - return ProcessingPipeline(
39 - name="Generic Log Sources to Sysmon Transformation",
40 - priority=10,
41 - items=[
42 - processing_item
43 - for log_source, event_id in sysmon_generic_logsource_eventid_mapping.items()
44 - for processing_item in (
45 - ProcessingItem(
46 - identifier=f"sysmon_{log_source}_eventid",
47 - transformation=AddConditionTransformation(
48 - {
49 - "EventID": event_id,
50 - },
51 - ),
52 - rule_conditions=[LogsourceCondition(category=log_source, product="windows")],
53 - ),
54 - ProcessingItem(
55 - identifier=f"sysmon_{log_source}_logsource",
56 - transformation=ChangeLogsourceTransformation(
57 - product="windows",
58 - service="sysmon",
59 - category=log_source,
60 - ),
61 - rule_conditions=[LogsourceCondition(category=log_source, product="windows")],
62 - ),
63 - )
64 - ],
65 - )
backend/app/connectors/wazuh_indexer/services/sigma/windows.py deleted
-265
@@ -1,265 +0,0 @@
1 -from sigma.pipelines.common import generate_windows_logsource_items
2 -from sigma.processing.conditions import FieldNameProcessingItemAppliedCondition
3 -from sigma.processing.conditions import IncludeFieldCondition
4 -from sigma.processing.conditions import LogsourceCondition
5 -from sigma.processing.pipeline import ProcessingItem
6 -from sigma.processing.pipeline import ProcessingPipeline
7 -from sigma.processing.transformations import AddFieldnamePrefixTransformation
8 -from sigma.processing.transformations import FieldMappingTransformation
9 -
10 -ecs_windows_variable_mappings = {
11 - "FileVersion": (
12 - ("category", "process_creation", "data_win_eventdata_fileVersion"),
13 - ("category", "image_load", "data_win_eventdata_fileVersion"),
14 - ),
15 - "Description": (
16 - ("category", "process_creation", "data_win_eventdata_description"),
17 - ("category", "image_load", "data_win_eventdata_description"),
18 - ("category", "sysmon_error", "winlog.event_data.Description"),
19 - ),
20 - "Product": (
21 - ("category", "process_creation", "data_win_eventdata_product"),
22 - ("category", "image_load", "data_win_eventdata_product"),
23 - ),
24 - "Company": (
25 - ("category", "process_creation", "data_win_eventdata_company"),
26 - ("category", "image_load", "data_win_eventdata_company"),
27 - ),
28 - "OriginalFileName": (
29 - ("category", "process_creation", "data_win_eventdata_image"),
30 - ("category", "process_creation", "data_win_eventdata_image"),
31 - ("category", "image_load", "data_win_eventdata_originalFileName"),
32 - ),
33 - "CommandLine": (
34 - ("category", "process_creation", "data_win_eventdata_commandLine"),
35 - ("service", "security", "data_win_eventdata_commandLine"),
36 - ("service", "powershell-classic", "powershell.command.value"),
37 - ),
38 - "Protocol": (("category", "network_connection", "data_win_eventdata_protocol"),),
39 - "Initiated": (("category", "network_connection", "data_win_eventdata_initiated"),),
40 - "Signature": (
41 - ("category", "driver_loaded", "data_win_eventdata_signatureSubjectName"),
42 - ("category", "image_loaded", "data_win_eventdata_signatureSubjectName"),
43 - ),
44 - "EngineVersion": (("service", "powershell-classic", "powershell.engine.version"),),
45 - "HostVersion": (("service", "powershell-classic", "powershell.process.executable_version"),),
46 - "SubjectLogonId": (("service", "security", "data_win_eventdata_subjectLogonId"),),
47 - "ServiceName": (("service", "security", "data_win_eventdata_serviceName"),),
48 - "SubjectDomainName": (("service", "security", "data_win_eventdata_subjectDomainName"),),
49 - "SubjectUserName": (("service", "security", "data_win_eventdata_subjectUserName"),),
50 - "SubjectUserSid": (("service", "security", "data_win_eventdata_subjectUserSid"),),
51 - "TargetLogonId": (("service", "security", "data_win_eventdata_targetLogonId"),),
52 -}
53 -
54 -
55 -def ecs_windows() -> ProcessingPipeline:
56 - return ProcessingPipeline(
57 - name="Elastic Common Schema (ECS) Windows log mappings from Winlogbeat from version 7",
58 - priority=20,
59 - allowed_backends=("elasticsearch", "eql", "lucene", "opensearch"),
60 - items=generate_windows_logsource_items("data_win_system_channel", "{source}")
61 - + [ # Variable field mapping depending on category/service
62 - ProcessingItem(
63 - identifier=f"elasticsearch_windows-{field}-{logsrc_field}-{logsrc}",
64 - transformation=FieldMappingTransformation({field: mapped}),
65 - rule_conditions=[
66 - LogsourceCondition(
67 - **{
68 - "product": "windows",
69 - logsrc_field: logsrc,
70 - },
71 - ),
72 - ],
73 - )
74 - for field, mappings in ecs_windows_variable_mappings.items()
75 - for (logsrc_field, logsrc, mapped) in mappings
76 - ]
77 - + [
78 - ProcessingItem( # Field mappings
79 - identifier="ecs_windows_field_mapping",
80 - transformation=FieldMappingTransformation(
81 - {
82 - "Accesses": "data_win_eventdata_accesses",
83 - "AccessList": "data_win_eventdata_accessList",
84 - "AccessMask": "data_win_eventdata_accessMask",
85 - "AccountName": "data_win_eventdata_targetUserName",
86 - "Action": "data_win_eventdata_action",
87 - "AllowedToDelegateTo": "data_win_eventdata_allowedToDelegateTo",
88 - "Application": "data_win_eventdata_application",
89 - "ApplicationPath": "data_win_eventdata_applicationPath",
90 - "AttributeLDAPDisplayName": "data_win_eventdata_attributeLDAPDisplayName",
91 - "AttributeValue": "data_win_eventdata_attributeValue",
92 - "AuditPolicyChanges": "data_win_evendata_auditPolicyChanges",
93 - "AuditSourceName": "data_win_eventdata_auditSourceName",
94 - "AuthenticationPackage": "data_win_eventdata_authenticationPackageName",
95 - "AuthenticationPackageName": "data_win_eventdata_authenticationPackageName",
96 - "CallTrace": "data_win_eventdata_callTrace",
97 - "Caption": "data_win_eventdata_caption",
98 - "Channel": "data_win_eventdata_channel",
99 - "ChildImage": "data_win_eventdata_image",
100 - "CommandLine": "data_win_eventdata_commandLine",
101 - "Company": "data_win_eventdata_company",
102 - "ComputerName": "data_win_system_computer",
103 - "ContextInfo": "data_win_system_contextInfo",
104 - "CurrentDirectory": "data_win_eventdata_currentDirectory",
105 - "Description": "data_win_eventdata_description",
106 - "DestAddress": "data_win_eventdata_destAddress",
107 - "Destination": "data_win_eventdata_destination",
108 - "DestinationHostname": "data_win_eventdata_destinationHostname",
109 - "DestinationIp": "data_win_eventdata_destinationIp",
110 - "DestinationIsIpv6": "data_win_eventdata_destinationIsIpv6",
111 - "DestinationPort": "data_win_eventdata_destinationPort",
112 - "Details": "data_win_eventdata_details",
113 - "DeviceClassName": "data_win_eventdata_deviceClassName",
114 - "DeviceDescription": "data_win_eventdata_deviceDescription",
115 - "DeviceName": "data_win_eventdata_deviceName",
116 - "DestPort": "data_win_eventdata_destinationPort",
117 - "EngineVersion": "data_win_eventdata_engineVersion",
118 - "EventID": "data_win_system_eventID",
119 - "EventType": "data_win_eventdata_eventType",
120 - "FailureCode": "data_win_eventdata_failureCode",
121 - "FileVersion": "data_win_eventdata_fileVersion",
122 - "FilterName": "data_win_evendata_filterName",
123 - "FilterOrigin": "data_win_eventdata_filterOrigin",
124 - "FolderPath": "data_win_eventdata_image",
125 - "GrantedAccess": "data_win_eventdata_grantedAccess",
126 - "Hash": "data_win_eventdata_hashes",
127 - "Hashes": "data_win_eventdata_hashes",
128 - "HostApplication": "data_win_eventdata_hostApplication",
129 - "HostName": "data_win_eventdata_hostName",
130 - "HostVersion": "data_win_eventdata_hostVersion",
131 - "Image": "data_win_eventdata_image",
132 - "ImageName": "data_win_evendata_imageName",
133 - "ImagePath": "data_win_eventdata_imagePath",
134 - "ImageLoaded": "data_win_eventdata_imageLoaded",
135 - "ImpHash": "data_win_eventdata_impHash",
136 - "Imphash": "data_win_eventdata_imphash",
137 - "ImpersonationLevel": "data_win_eventdata_impersonationLevel",
138 - "Initiated": "data_win_eventdata_initiated",
139 - "IntegrityLevel": "data_win_eventdata_integrityLevel",
140 - "IpAddress": "data_win_eventdata_ipAddress",
141 - "KeyLength": "data_win_eventdata_keyLength",
142 - "Keywords": "data_win_eventdata_keywords",
143 - "LayerRTID": "data_win_eventdata_layerRTID",
144 - "Level": "data_win_system_level",
145 - "LogonGuid": "data_win_eventdata_logonGuid",
146 - "LogonId": "data_win_eventdata_logonId",
147 - "LogonProcessName": "data_win_eventdata_logonProcessName",
148 - "LogonType": "data_win_eventdata_logonType",
149 - "md5": "data_win_eventdata_hashes",
150 - "Message": "data_win_system_message",
151 - "ModifyingApplication": "data_win_system_modifyingApplication",
152 - "NewName": "data_win_eventdata_newName",
153 - "NewTargetUserName": "data_win_evendata_newTargetUserName",
154 - "NewUacValue": "data_win_eventdata_newUacValue",
155 - "NewValue": "data_win_eventdata_newValue",
156 - "ObjectClass": "data_win_eventdata_objectClass",
157 - "ObjectName": "data_win_eventdata_objectName",
158 - "ObjectServer": "data_win_eventdata_objectServer",
159 - "ObjectType": "data_win_eventdata_objectType",
160 - "ObjectValueName": "data_win_eventdata_objectValueName",
161 - "OldUacValue": "data_win_eventdata_oldUacValue",
162 - "Origin": "data_win_eventdata_origin",
163 - "OriginalFileName": "data_win_eventdata_originalFileName",
164 - "PackageName": "data_win_eventdata_packageName",
165 - "Param1": "data_win_evendata_param1",
166 - "Param2": "data_win_evendata_param2",
167 - "Param3": "data_win_evendata_param3",
168 - "Param4": "data_win_evendata_Param4",
169 - "Param5": "data_win_evendata_param5",
170 - "Param6": "data_win_evendata_param6",
171 - "Param7": "data_win_evendata_param7",
172 - "Param8": "data_win_evendata_Param8",
173 - "Param9": "data_win_evendata_Param9",
174 - "Param10": "data_win_evendata_Param10",
175 - "ParentCommandLine": "data_win_eventdata_parentCommandLine",
176 - "ParentImage": "data_win_eventdata_parentImage",
177 - "ParentIntegrityLevel": "data_win_eventdata_parentIntegrityLevel",
178 - "ParentProcessGuid": "data_win_eventdata_parentProcessGuid",
179 - "ParentUser": "data_win_eventdata_parentUser",
180 - "Payload": "data_win_eventdata_payload",
181 - "PipeName": "data_win_eventdata_pipeName",
182 - "PrivilegeList": "data_win_eventdata_privilegeList",
183 - "ProcessCommandLine": "data_win_eventdata_commandLine",
184 - "ProcessID": "data_win_eventdata_processId",
185 - "ProcessName": "data_win_eventdata_processName",
186 - "ProcessPath": "data_win_eventdata_processPath",
187 - "Product": "data_win_eventdata_product",
188 - "Properties": "data_win_eventdata_properties",
189 - "ProviderContextName": "data_win_evendata_providerContextName",
190 - "ProviderName": "data_win_eventdata_providerName",
191 - "Provider_Name": "data_win_eventdata_providerName",
192 - "QueryName": "data_win_eventdata_queryName",
193 - "RelativeTargetName": "data_win_eventdata_relativeTargetName",
194 - "RemoteAddress": "data_win_eventdata_remoteAddress",
195 - "SamAccountName": "data_win_eventdata_samAccountName",
196 - "ScriptBlockText": "data_win_eventdata_scriptBlockText",
197 - "Service": "data_win_eventdata_service",
198 - "ServerName": "data_win_eventdata_serverName",
199 - "ServiceFileName": "data_win_eventdata_serviceFileName",
200 - "ServiceName": "data_win_eventdata_serviceName",
201 - "ServiceStartType": "data_win_evendata_serviceStartType",
202 - "ServiceType": "data_win_evendata_serviceType",
203 - "sha1": "data_win_eventdata_hashes",
204 - "sha256": "data_win_eventdata_hashes",
205 - "ShareName": "data_win_eventdata_shareName",
206 - "SidHistory": "data_win_eventdata_sidHistory",
207 - "Signed": "data_win_eventdata_signed",
208 - "Source": "data_win_eventdata_source",
209 - "Source_Name": "data_win_eventdata_sourceName",
210 - "SourceAddress": "data_win_eventdata_sourceAddress",
211 - "SourceImage": "data_win_eventdata_sourceImage",
212 - "SourceNetworkAddress": "data_win_eventdata_ipAddress",
213 - "TargetImage": "data_win_eventdata_targetImage",
214 - "TargetObject": "data_win_eventdata_targetObject",
215 - "TargetUser": "data_win_eventdata_targetUser",
216 - },
217 - ),
218 - rule_conditions=[LogsourceCondition(product="windows")],
219 - ),
220 - ProcessingItem( # Prepend each field that was not processed by previous field mapping transformation with "winlog.event_data."
221 - identifier="ecs_windows_winlog_eventdata_prefix",
222 - # transformation=AddFieldnamePrefixTransformation("winlog.event_data."),
223 - transformation=AddFieldnamePrefixTransformation(""),
224 - field_name_conditions=[
225 - FieldNameProcessingItemAppliedCondition("ecs_windows_field_mapping"),
226 - IncludeFieldCondition(fields=["\\w+\\."], type="re"),
227 - ],
228 - field_name_condition_negation=True,
229 - field_name_condition_linking=any,
230 - rule_conditions=[LogsourceCondition(product="windows")],
231 - ),
232 - ],
233 - )
234 -
235 -
236 -def ecs_windows_old() -> ProcessingPipeline:
237 - return ProcessingPipeline(
238 - name="Elastic Common Schema (ECS) Windows log mappings from Winlogbeat up to version 6",
239 - priority=20,
240 - allowed_backends=("elasticsearch", "eql", "lucene", "opensearch"),
241 - items=generate_windows_logsource_items("winlog.channel", "{source}")
242 - + [
243 - ProcessingItem( # Field mappings
244 - identifier="ecs_windows_field_mapping",
245 - transformation=FieldMappingTransformation(
246 - {
247 - "EventID": "event_id",
248 - "Channel": "winlog.channel",
249 - },
250 - ),
251 - rule_conditions=[LogsourceCondition(product="windows")],
252 - ),
253 - ProcessingItem( # Prepend each field that was not processed by previous field mapping transformation with "winlog.event_data."
254 - identifier="ecs_windows_eventdata_prefix",
255 - transformation=AddFieldnamePrefixTransformation("event_data."),
256 - field_name_conditions=[
257 - FieldNameProcessingItemAppliedCondition("ecs_windows_field_mapping"),
258 - IncludeFieldCondition(fields=["\\w+\\."], type="re"),
259 - ],
260 - field_name_condition_negation=True,
261 - field_name_condition_linking=any,
262 - rule_conditions=[LogsourceCondition(product="windows")],
263 - ),
264 - ],
265 - )
backend/app/routers/wazuh_indexer.py
-6
@@ -2,7 +2,6 @@ from fastapi import APIRouter
2
3 from app.connectors.wazuh_indexer.routes.alerts import wazuh_indexer_alerts_router
4 from app.connectors.wazuh_indexer.routes.monitoring import wazuh_indexer_router
5 -from app.connectors.wazuh_indexer.routes.sigma import wazuh_indexer_sigma_router
5 from app.connectors.wazuh_indexer.routes.snapshot_and_restore import (
6 wazuh_indexer_snapshots_router,
7 )
@@ -21,11 +20,6 @@ router.include_router(
20 prefix="/wazuh_indexer",
21 tags=["wazuh-indexer-monitoring"],
22 )
24 -router.include_router(
25 - wazuh_indexer_sigma_router,
26 - prefix="/sigma",
27 - tags=["wazuh-indexer-sigma"],
28 -)
23 router.include_router(
24 wazuh_indexer_snapshots_router,
25 prefix="/snapshots",
backend/app/schedulers/scheduler.py
-3
@@ -62,7 +62,6 @@ from app.schedulers.services.invoke_sap_siem import (
62 from app.schedulers.services.invoke_sap_siem import (
63 invoke_sap_siem_integration_suspicious_logins_analysis,
64 )
65 -from app.schedulers.services.invoke_sigma_queries import invoke_sigma_queries_collect
65 from app.schedulers.services.invoke_snapshot_and_restore import (
66 invoke_snapshot_schedules,
67 )
@@ -233,7 +232,6 @@ async def schedule_enabled_jobs(scheduler):
232 "wazuh_index_fields_resize",
233 "invoke_huntress_integration_collection",
234 "invoke_cato_integration_collect",
236 - "invoke_sigma_queries_collect",
235 ]
236
237 # Disable each job in the list
@@ -283,7 +281,6 @@ def get_function_by_name(function_name: str):
281 # "wazuh_index_fields_resize": resize_wazuh_index_fields,
282 "resize_wazuh_index_fields": resize_wazuh_index_fields,
283 "invoke_alert_creation_collect": invoke_alert_creation_collect,
286 - "invoke_sigma_queries_collect": invoke_sigma_queries_collect,
284 "invoke_snapshot_schedules": invoke_snapshot_schedules,
285 "invoke_mimecast_integration": invoke_mimecast_integration,
286 "invoke_mimecast_integration_ttp": invoke_mimecast_integration_ttp,
backend/app/schedulers/services/invoke_sigma_queries.py deleted
-32
@@ -1,32 +0,0 @@
1 -from datetime import datetime
2 -
3 -from loguru import logger
4 -from sqlalchemy.future import select
5 -
6 -from app.connectors.wazuh_indexer.routes.sigma import run_active_sigma_queries_endpoint
7 -from app.db.db_session import get_db_session
8 -from app.schedulers.models.scheduler import JobMetadata
9 -
10 -
11 -async def invoke_sigma_queries_collect():
12 - """
13 - Invokes the analysis of Sigma enabled queries via the scheduler.
14 -
15 - If the token retrieval fails, it prints a failure message. If the job metadata for
16 - 'invoke_sigma_queries_collect' does not exist, it prints a message indicating the absence of the metadata.
17 - """
18 - logger.info("Invoking sigma queries collection via scheduler...")
19 - async with get_db_session() as session:
20 - await run_active_sigma_queries_endpoint(index_name="wazuh*", db=session)
21 -
22 - stmt = select(JobMetadata).where(JobMetadata.job_id == "invoke_sigma_queries_collect")
23 - result = await session.execute(stmt)
24 - job_metadata = result.scalars().first()
25 -
26 - if job_metadata:
27 - job_metadata.last_success = datetime.utcnow()
28 - session.add(job_metadata)
29 - await session.commit() # Asynchronously commit the transaction
30 - logger.info("Updated job metadata with the last success timestamp.")
31 - else:
32 - logger.warning("JobMetadata for 'invoke_sigma_queries_collect' not found.")