| 1 | import asyncio |
| 2 | import copy |
| 3 | import json |
| 4 | import os |
| 5 | import re |
| 6 | from datetime import datetime |
| 7 | from datetime import timedelta |
| 8 | from typing import Any |
| 9 | from typing import Optional |
| 10 | |
| 11 | import httpx |
| 12 | import yaml |
| 13 | from loguru import logger |
| 14 | |
| 15 | |
| 16 | def _github_headers() -> dict[str, str]: |
| 17 | headers = {"Accept": "application/vnd.github+json"} |
| 18 | token = os.getenv("GITHUB_TOKEN") |
| 19 | if token: |
| 20 | headers["Authorization"] = f"Bearer {token}" |
| 21 | return headers |
| 22 | |
| 23 | |
| 24 | from app.connectors.wazuh_indexer.utils.universal import ( |
| 25 | create_wazuh_indexer_client_async, |
| 26 | ) |
| 27 | from app.integrations.copilot_searches.schema.copilot_searches import ( |
| 28 | ExecuteGraylogQueryRequest, |
| 29 | ) |
| 30 | from app.integrations.copilot_searches.schema.copilot_searches import ( |
| 31 | ExecuteSearchRequest, |
| 32 | ) |
| 33 | from app.integrations.copilot_searches.schema.copilot_searches import ( |
| 34 | ExecuteSearchResponse, |
| 35 | ) |
| 36 | from app.integrations.copilot_searches.schema.copilot_searches import GraylogQuery |
| 37 | from app.integrations.copilot_searches.schema.copilot_searches import ( |
| 38 | GraylogQueryResponse, |
| 39 | ) |
| 40 | from app.integrations.copilot_searches.schema.copilot_searches import ParameterSchema |
| 41 | from app.integrations.copilot_searches.schema.copilot_searches import PlatformFilter |
| 42 | from app.integrations.copilot_searches.schema.copilot_searches import ( |
| 43 | ProvisionGraylogAlertRequest, |
| 44 | ) |
| 45 | from app.integrations.copilot_searches.schema.copilot_searches import ( |
| 46 | ProvisionGraylogAlertResponse, |
| 47 | ) |
| 48 | from app.integrations.copilot_searches.schema.copilot_searches import RuleDetail |
| 49 | from app.integrations.copilot_searches.schema.copilot_searches import RuleSeverity |
| 50 | from app.integrations.copilot_searches.schema.copilot_searches import RuleStatus |
| 51 | from app.integrations.copilot_searches.schema.copilot_searches import RuleSummary |
| 52 | from app.integrations.copilot_searches.schema.copilot_searches import SearchHit |
| 53 | from app.integrations.copilot_searches.schema.copilot_searches import ( |
| 54 | SearchValidationError, |
| 55 | ) |
| 56 | from app.integrations.monitoring_alert.schema.provision import ( |
| 57 | GraylogAlertProvisionConfig, |
| 58 | ) |
| 59 | from app.integrations.monitoring_alert.schema.provision import ( |
| 60 | GraylogAlertProvisionFieldSpecItem, |
| 61 | ) |
| 62 | from app.integrations.monitoring_alert.schema.provision import ( |
| 63 | GraylogAlertProvisionModel, |
| 64 | ) |
| 65 | from app.integrations.monitoring_alert.schema.provision import ( |
| 66 | GraylogAlertProvisionNotificationSettings, |
| 67 | ) |
| 68 | from app.integrations.monitoring_alert.schema.provision import ( |
| 69 | GraylogAlertProvisionProvider, |
| 70 | ) |
| 71 | from app.integrations.monitoring_alert.services.provision import ( |
| 72 | provision_alert_definition, |
| 73 | ) |
| 74 | |
| 75 | # ============================================================================= |
| 76 | # Configuration |
| 77 | # ============================================================================= |
| 78 | |
| 79 | GITHUB_REPO = "socfortress/CoPilot-Search-Queries" |
| 80 | GITHUB_BRANCH = "main" |
| 81 | GITHUB_API_BASE = "https://api.github.com" |
| 82 | GITHUB_RAW_BASE = "https://raw.githubusercontent.com" |
| 83 | |
| 84 | # Cache settings |
| 85 | CACHE_TTL_MINUTES = 30 |
| 86 | |
| 87 | |
| 88 | # ============================================================================= |
| 89 | # Rules Cache |
| 90 | # ============================================================================= |
| 91 | |
| 92 | |
| 93 | class RulesCache: |
| 94 | """ |
| 95 | In-memory cache for detection rules fetched from GitHub. |
| 96 | |
| 97 | Handles fetching, parsing, and caching YAML rules from the repository. |
| 98 | """ |
| 99 | |
| 100 | def __init__(self): |
| 101 | self._rules: dict[str, dict] = {} # id -> rule data |
| 102 | self._rules_by_name: dict[str, str] = {} # normalized name -> id |
| 103 | self._last_refresh: Optional[datetime] = None |
| 104 | self._lock = asyncio.Lock() |
| 105 | |
| 106 | @property |
| 107 | def is_stale(self) -> bool: |
| 108 | """Check if cache needs refresh.""" |
| 109 | if self._last_refresh is None: |
| 110 | return True |
| 111 | age = datetime.utcnow() - self._last_refresh |
| 112 | return age > timedelta(minutes=CACHE_TTL_MINUTES) |
| 113 | |
| 114 | @property |
| 115 | def cache_age_minutes(self) -> Optional[float]: |
| 116 | """Get cache age in minutes.""" |
| 117 | if self._last_refresh is None: |
| 118 | return None |
| 119 | age = datetime.utcnow() - self._last_refresh |
| 120 | return age.total_seconds() / 60 |
| 121 | |
| 122 | @property |
| 123 | def last_refresh(self) -> Optional[datetime]: |
| 124 | """Get last refresh timestamp.""" |
| 125 | return self._last_refresh |
| 126 | |
| 127 | @property |
| 128 | def rules_count(self) -> int: |
| 129 | """Get number of cached rules.""" |
| 130 | return len(self._rules) |
| 131 | |
| 132 | async def ensure_loaded(self): |
| 133 | """Ensure rules are loaded, refreshing if stale.""" |
| 134 | if self.is_stale: |
| 135 | await self.refresh() |
| 136 | |
| 137 | async def refresh(self) -> int: |
| 138 | """ |
| 139 | Refresh rules cache from GitHub repository. |
| 140 | |
| 141 | Returns: |
| 142 | Number of rules loaded |
| 143 | """ |
| 144 | async with self._lock: |
| 145 | logger.info("Refreshing rules cache from GitHub...") |
| 146 | |
| 147 | rules = await self._fetch_all_rules() |
| 148 | |
| 149 | self._rules = {} |
| 150 | self._rules_by_name = {} |
| 151 | |
| 152 | for rule in rules: |
| 153 | rule_id = rule.get("id", "") |
| 154 | rule_name = rule.get("name", "") |
| 155 | |
| 156 | self._rules[rule_id] = rule |
| 157 | |
| 158 | # Index by normalized name for lookup |
| 159 | normalized_name = self._normalize_name(rule_name) |
| 160 | self._rules_by_name[normalized_name] = rule_id |
| 161 | |
| 162 | self._last_refresh = datetime.utcnow() |
| 163 | logger.info(f"Loaded {len(self._rules)} rules from GitHub") |
| 164 | |
| 165 | return len(self._rules) |
| 166 | |
| 167 | async def _fetch_all_rules(self) -> list[dict]: |
| 168 | """Fetch all YAML rules from GitHub repository.""" |
| 169 | rules = [] |
| 170 | |
| 171 | async with httpx.AsyncClient(timeout=30.0, headers=_github_headers()) as client: |
| 172 | # Get the directory tree for detections |
| 173 | tree_url = f"{GITHUB_API_BASE}/repos/{GITHUB_REPO}/git/trees/{GITHUB_BRANCH}" f"?recursive=1" |
| 174 | |
| 175 | response = await client.get(tree_url) |
| 176 | response.raise_for_status() |
| 177 | |
| 178 | tree_data = response.json() |
| 179 | |
| 180 | # Filter for YAML files in detections directory |
| 181 | yaml_files = [ |
| 182 | item |
| 183 | for item in tree_data.get("tree", []) |
| 184 | if item["path"].startswith("detections/") and item["path"].endswith(".yaml") and item["type"] == "blob" |
| 185 | ] |
| 186 | |
| 187 | logger.info(f"Found {len(yaml_files)} YAML files in repository") |
| 188 | |
| 189 | # Fetch each YAML file |
| 190 | tasks = [self._fetch_yaml_file(client, file_info["path"]) for file_info in yaml_files] |
| 191 | |
| 192 | results = await asyncio.gather(*tasks, return_exceptions=True) |
| 193 | |
| 194 | for result in results: |
| 195 | if isinstance(result, Exception): |
| 196 | logger.warning(f"Failed to fetch rule: {result}") |
| 197 | elif result is not None: |
| 198 | rules.append(result) |
| 199 | |
| 200 | return rules |
| 201 | |
| 202 | async def _fetch_yaml_file( |
| 203 | self, |
| 204 | client: httpx.AsyncClient, |
| 205 | file_path: str, |
| 206 | ) -> Optional[dict]: |
| 207 | """Fetch and parse a single YAML file from GitHub.""" |
| 208 | try: |
| 209 | raw_url = f"{GITHUB_RAW_BASE}/{GITHUB_REPO}/{GITHUB_BRANCH}/{file_path}" |
| 210 | |
| 211 | response = await client.get(raw_url) |
| 212 | response.raise_for_status() |
| 213 | |
| 214 | raw_yaml = response.text |
| 215 | rule_data = yaml.safe_load(raw_yaml) |
| 216 | |
| 217 | if not isinstance(rule_data, dict): |
| 218 | logger.warning(f"Invalid YAML structure in {file_path}") |
| 219 | return None |
| 220 | |
| 221 | # Add metadata |
| 222 | rule_data["_file_path"] = file_path |
| 223 | rule_data["_raw_yaml"] = raw_yaml |
| 224 | rule_data["_platform"] = self._detect_platform(file_path, rule_data) |
| 225 | rule_data["_has_graylog"] = "graylog" in rule_data and bool(rule_data.get("graylog", {}).get("query")) |
| 226 | |
| 227 | return rule_data |
| 228 | |
| 229 | except Exception as e: |
| 230 | logger.warning(f"Error fetching {file_path}: {e}") |
| 231 | return None |
| 232 | |
| 233 | def _detect_platform(self, file_path: str, rule_data: dict) -> str: |
| 234 | """Detect the platform (Linux/Windows) for a rule.""" |
| 235 | path_lower = file_path.lower() |
| 236 | |
| 237 | # Check path first |
| 238 | if "/linux/" in path_lower: |
| 239 | return "linux" |
| 240 | if "/windows/" in path_lower: |
| 241 | return "windows" |
| 242 | if "/powershell/" in path_lower: |
| 243 | return "powershell" |
| 244 | if "/cve/" in path_lower: |
| 245 | return "cve" |
| 246 | |
| 247 | # Check tags |
| 248 | asset_type = rule_data.get("tags", {}).get("asset_type", "").lower() |
| 249 | if "linux" in asset_type: |
| 250 | return "linux" |
| 251 | if "windows" in asset_type: |
| 252 | return "windows" |
| 253 | |
| 254 | # Check rule name |
| 255 | name_lower = rule_data.get("name", "").lower() |
| 256 | if "powershell" in name_lower: |
| 257 | return "powershell" |
| 258 | if "cve" in name_lower: |
| 259 | return "cve" |
| 260 | if "linux" in name_lower: |
| 261 | return "linux" |
| 262 | if "windows" in name_lower: |
| 263 | return "windows" |
| 264 | |
| 265 | return "unknown" |
| 266 | |
| 267 | def _normalize_name(self, name: str) -> str: |
| 268 | """Normalize rule name for lookup.""" |
| 269 | return name.lower().strip().replace(" ", "_").replace("-", "_") |
| 270 | |
| 271 | def get_all_rules(self) -> list[dict]: |
| 272 | """Get all cached rules.""" |
| 273 | return list(self._rules.values()) |
| 274 | |
| 275 | def get_rule_by_id(self, rule_id: str) -> Optional[dict]: |
| 276 | """Get a rule by its ID.""" |
| 277 | return self._rules.get(rule_id) |
| 278 | |
| 279 | def get_rule_by_name(self, name: str) -> Optional[dict]: |
| 280 | """Get a rule by its name (fuzzy match).""" |
| 281 | normalized = self._normalize_name(name) |
| 282 | |
| 283 | # Exact match |
| 284 | if normalized in self._rules_by_name: |
| 285 | rule_id = self._rules_by_name[normalized] |
| 286 | return self._rules.get(rule_id) |
| 287 | |
| 288 | # Partial match |
| 289 | for stored_name, rule_id in self._rules_by_name.items(): |
| 290 | if normalized in stored_name or stored_name in normalized: |
| 291 | return self._rules.get(rule_id) |
| 292 | |
| 293 | return None |
| 294 | |
| 295 | def filter_rules( |
| 296 | self, |
| 297 | platform: PlatformFilter = PlatformFilter.ALL, |
| 298 | status: Optional[RuleStatus] = None, |
| 299 | severity: Optional[RuleSeverity] = None, |
| 300 | mitre_id: Optional[str] = None, |
| 301 | search: Optional[str] = None, |
| 302 | has_graylog: Optional[bool] = None, |
| 303 | ) -> list[dict]: |
| 304 | """Filter rules based on criteria.""" |
| 305 | results = [] |
| 306 | |
| 307 | for rule in self._rules.values(): |
| 308 | # Platform filter |
| 309 | if platform != PlatformFilter.ALL: |
| 310 | rule_platform = rule.get("_platform", "unknown") |
| 311 | if rule_platform != platform.value: |
| 312 | continue |
| 313 | |
| 314 | # Status filter |
| 315 | if status is not None: |
| 316 | rule_status = rule.get("status", "").lower() |
| 317 | if rule_status != status.value: |
| 318 | continue |
| 319 | |
| 320 | # Severity filter |
| 321 | if severity is not None: |
| 322 | rule_severity = rule.get("response", {}).get("severity", "").lower() |
| 323 | if rule_severity != severity.value: |
| 324 | continue |
| 325 | |
| 326 | # MITRE ATT&CK filter |
| 327 | if mitre_id is not None: |
| 328 | mitre_ids = rule.get("tags", {}).get("mitre_attack_id", []) |
| 329 | if not any(mitre_id.upper() in m.upper() for m in mitre_ids): |
| 330 | continue |
| 331 | |
| 332 | # Text search (name, description) |
| 333 | if search is not None: |
| 334 | search_lower = search.lower() |
| 335 | name = rule.get("name", "").lower() |
| 336 | description = rule.get("description", "").lower() |
| 337 | if search_lower not in name and search_lower not in description: |
| 338 | continue |
| 339 | |
| 340 | # Graylog query filter |
| 341 | if has_graylog is not None: |
| 342 | rule_has_graylog = rule.get("_has_graylog", False) |
| 343 | if rule_has_graylog != has_graylog: |
| 344 | continue |
| 345 | |
| 346 | results.append(rule) |
| 347 | |
| 348 | return results |
| 349 | |
| 350 | def get_stats(self) -> dict: |
| 351 | """Get statistics about cached rules.""" |
| 352 | stats = { |
| 353 | "total_rules": len(self._rules), |
| 354 | "by_platform": {}, |
| 355 | "by_status": {}, |
| 356 | "by_severity": {}, |
| 357 | "by_mitre_tactic": {}, |
| 358 | "rules_with_graylog": 0, |
| 359 | } |
| 360 | |
| 361 | for rule in self._rules.values(): |
| 362 | # By platform |
| 363 | platform = rule.get("_platform", "unknown") |
| 364 | stats["by_platform"][platform] = stats["by_platform"].get(platform, 0) + 1 |
| 365 | |
| 366 | # By status |
| 367 | status = rule.get("status", "unknown") |
| 368 | stats["by_status"][status] = stats["by_status"].get(status, 0) + 1 |
| 369 | |
| 370 | # By severity |
| 371 | severity = rule.get("response", {}).get("severity", "unknown") |
| 372 | stats["by_severity"][severity] = stats["by_severity"].get(severity, 0) + 1 |
| 373 | |
| 374 | # By MITRE tactic (extract tactic from technique ID) |
| 375 | mitre_ids = rule.get("tags", {}).get("mitre_attack_id", []) |
| 376 | for mitre_id in mitre_ids: |
| 377 | # Extract base technique (e.g., T1136 from T1136.001) |
| 378 | base_technique = mitre_id.split(".")[0] if "." in mitre_id else mitre_id |
| 379 | stats["by_mitre_tactic"][base_technique] = stats["by_mitre_tactic"].get(base_technique, 0) + 1 |
| 380 | |
| 381 | # Count rules with Graylog queries |
| 382 | if rule.get("_has_graylog", False): |
| 383 | stats["rules_with_graylog"] += 1 |
| 384 | |
| 385 | return stats |
| 386 | |
| 387 | |
| 388 | # ============================================================================= |
| 389 | # Helper Functions |
| 390 | # ============================================================================= |
| 391 | |
| 392 | |
| 393 | def rule_to_summary(rule: dict) -> RuleSummary: |
| 394 | """Convert a raw rule dict to a RuleSummary model.""" |
| 395 | tags = rule.get("tags", {}) |
| 396 | response = rule.get("response", {}) |
| 397 | |
| 398 | return RuleSummary( |
| 399 | id=rule.get("id", ""), |
| 400 | name=rule.get("name", ""), |
| 401 | version=rule.get("version", 1), |
| 402 | status=rule.get("status", "unknown"), |
| 403 | type=rule.get("type", "unknown"), |
| 404 | description=rule.get("description", ""), |
| 405 | author=rule.get("author", ""), |
| 406 | date=rule.get("date", ""), |
| 407 | severity=response.get("severity", "medium"), |
| 408 | risk_score=response.get("risk_score", 0), |
| 409 | platform=rule.get("_platform", "unknown"), |
| 410 | mitre_attack_id=tags.get("mitre_attack_id", []), |
| 411 | analytic_story=tags.get("analytic_story", []), |
| 412 | cve=tags.get("cve", []), |
| 413 | file_path=rule.get("_file_path", ""), |
| 414 | has_graylog_query=rule.get("_has_graylog", False), |
| 415 | ) |
| 416 | |
| 417 | |
| 418 | def rule_to_detail(rule: dict) -> RuleDetail: |
| 419 | """Convert a raw rule dict to a RuleDetail model.""" |
| 420 | # Parse parameters |
| 421 | params = [] |
| 422 | for name, param_data in rule.get("parameters", {}).items(): |
| 423 | params.append( |
| 424 | ParameterSchema( |
| 425 | name=name, |
| 426 | description=param_data.get("description", ""), |
| 427 | type=param_data.get("type", "string"), |
| 428 | required=param_data.get("required", False), |
| 429 | default=param_data.get("default"), |
| 430 | example=param_data.get("example"), |
| 431 | ), |
| 432 | ) |
| 433 | |
| 434 | # Parse Graylog query if present |
| 435 | graylog = None |
| 436 | graylog_data = rule.get("graylog") |
| 437 | if graylog_data and isinstance(graylog_data, dict) and graylog_data.get("query"): |
| 438 | graylog = GraylogQuery(query=graylog_data.get("query", "")) |
| 439 | |
| 440 | return RuleDetail( |
| 441 | id=rule.get("id", ""), |
| 442 | name=rule.get("name", ""), |
| 443 | version=rule.get("version", 1), |
| 444 | schema_version=rule.get("schema_version", "1.0"), |
| 445 | status=rule.get("status", "unknown"), |
| 446 | type=rule.get("type", "unknown"), |
| 447 | description=rule.get("description", ""), |
| 448 | author=rule.get("author", ""), |
| 449 | date=rule.get("date", ""), |
| 450 | data_source=rule.get("data_source", []), |
| 451 | search=rule.get("search", {}), |
| 452 | parameters=params, |
| 453 | how_to_implement=rule.get("how_to_implement", ""), |
| 454 | known_false_positives=rule.get("known_false_positives", ""), |
| 455 | references=rule.get("references", []), |
| 456 | response=rule.get("response", {}), |
| 457 | tags=rule.get("tags", {}), |
| 458 | file_path=rule.get("_file_path", ""), |
| 459 | raw_yaml=rule.get("_raw_yaml", ""), |
| 460 | graylog=graylog, |
| 461 | ) |
| 462 | |
| 463 | |
| 464 | def _convert_seconds_to_milliseconds(seconds: int) -> int: |
| 465 | """Convert seconds to milliseconds.""" |
| 466 | return seconds * 1000 |
| 467 | |
| 468 | |
| 469 | def _get_alert_source_from_rule(rule: dict) -> str: |
| 470 | """ |
| 471 | Determine the alert source based on rule metadata. |
| 472 | |
| 473 | Args: |
| 474 | rule: The rule dictionary |
| 475 | |
| 476 | Returns: |
| 477 | Alert source string (e.g., "WAZUH", "LINUX_AUDITD", etc.) |
| 478 | """ |
| 479 | # Check data_source field |
| 480 | data_sources = rule.get("data_source", []) |
| 481 | if data_sources: |
| 482 | # Use first data source, normalized |
| 483 | source = data_sources[0].upper().replace(" ", "_").replace("-", "_") |
| 484 | return source |
| 485 | |
| 486 | # Check platform |
| 487 | platform = rule.get("_platform", "unknown") |
| 488 | if platform == "linux": |
| 489 | return "LINUX" |
| 490 | if platform == "windows": |
| 491 | return "WINDOWS" |
| 492 | |
| 493 | return "COPILOT_SEARCH" |
| 494 | |
| 495 | |
| 496 | def _get_priority_from_severity(severity: str) -> int: |
| 497 | """ |
| 498 | Map rule severity to Graylog priority. |
| 499 | |
| 500 | Args: |
| 501 | severity: Rule severity (low, medium, high, critical) |
| 502 | |
| 503 | Returns: |
| 504 | Graylog priority (1=Low, 2=Normal, 3=High) |
| 505 | """ |
| 506 | severity_map = { |
| 507 | "low": 1, |
| 508 | "medium": 2, |
| 509 | "high": 3, |
| 510 | "critical": 3, |
| 511 | } |
| 512 | return severity_map.get(severity.lower(), 2) |
| 513 | |
| 514 | |
| 515 | # ============================================================================= |
| 516 | # Global Cache Instance |
| 517 | # ============================================================================= |
| 518 | |
| 519 | rules_cache = RulesCache() |
| 520 | |
| 521 | |
| 522 | # ============================================================================= |
| 523 | # Service Functions |
| 524 | # ============================================================================= |
| 525 | |
| 526 | |
| 527 | async def get_rules_list( |
| 528 | platform: PlatformFilter = PlatformFilter.ALL, |
| 529 | status: Optional[RuleStatus] = None, |
| 530 | severity: Optional[RuleSeverity] = None, |
| 531 | mitre_id: Optional[str] = None, |
| 532 | search: Optional[str] = None, |
| 533 | has_graylog: Optional[bool] = None, |
| 534 | skip: int = 0, |
| 535 | limit: int = 100, |
| 536 | ) -> dict: |
| 537 | """ |
| 538 | Get filtered list of detection rules. |
| 539 | |
| 540 | Args: |
| 541 | platform: Filter by platform (linux, windows, powershell, all) |
| 542 | status: Filter by rule status |
| 543 | severity: Filter by severity level |
| 544 | mitre_id: Filter by MITRE ATT&CK technique ID |
| 545 | search: Text search in name/description |
| 546 | has_graylog: Filter for rules with Graylog queries |
| 547 | skip: Number of rules to skip |
| 548 | limit: Maximum rules to return |
| 549 | |
| 550 | Returns: |
| 551 | Dictionary with total, filtered count, platform, and rules list |
| 552 | """ |
| 553 | await rules_cache.ensure_loaded() |
| 554 | |
| 555 | # Filter rules |
| 556 | filtered_rules = rules_cache.filter_rules( |
| 557 | platform=platform, |
| 558 | status=status, |
| 559 | severity=severity, |
| 560 | mitre_id=mitre_id, |
| 561 | search=search, |
| 562 | has_graylog=has_graylog, |
| 563 | ) |
| 564 | |
| 565 | # Sort by name |
| 566 | filtered_rules.sort(key=lambda r: r.get("name", "").lower()) |
| 567 | |
| 568 | # Paginate |
| 569 | total_filtered = len(filtered_rules) |
| 570 | paginated = filtered_rules[skip : skip + limit] |
| 571 | |
| 572 | # Convert to summaries |
| 573 | summaries = [rule_to_summary(rule) for rule in paginated] |
| 574 | |
| 575 | return { |
| 576 | "total": rules_cache.rules_count, |
| 577 | "filtered": total_filtered, |
| 578 | "platform": platform.value, |
| 579 | "rules": summaries, |
| 580 | } |
| 581 | |
| 582 | |
| 583 | async def get_rule_by_id(rule_id: str) -> Optional[RuleDetail]: |
| 584 | """ |
| 585 | Get full details of a rule by its ID. |
| 586 | |
| 587 | Args: |
| 588 | rule_id: The rule ID |
| 589 | |
| 590 | Returns: |
| 591 | RuleDetail or None if not found |
| 592 | """ |
| 593 | await rules_cache.ensure_loaded() |
| 594 | |
| 595 | rule = rules_cache.get_rule_by_id(rule_id) |
| 596 | |
| 597 | if rule is None: |
| 598 | return None |
| 599 | |
| 600 | return rule_to_detail(rule) |
| 601 | |
| 602 | |
| 603 | async def get_rules_by_ids(ids: list[str]) -> tuple[list[RuleSummary], list[str]]: |
| 604 | """ |
| 605 | Fetch many rule summaries by ID in one shot. |
| 606 | |
| 607 | Returns (found_summaries, missing_ids). |
| 608 | """ |
| 609 | await rules_cache.ensure_loaded() |
| 610 | |
| 611 | found: list[RuleSummary] = [] |
| 612 | missing: list[str] = [] |
| 613 | for rule_id in ids: |
| 614 | rule = rules_cache.get_rule_by_id(rule_id) |
| 615 | if rule is None: |
| 616 | missing.append(rule_id) |
| 617 | else: |
| 618 | found.append(rule_to_summary(rule)) |
| 619 | return found, missing |
| 620 | |
| 621 | |
| 622 | async def get_rule_by_name(rule_name: str) -> Optional[RuleDetail]: |
| 623 | """ |
| 624 | Get full details of a rule by its name (fuzzy match). |
| 625 | |
| 626 | Args: |
| 627 | rule_name: The rule name (supports fuzzy matching) |
| 628 | |
| 629 | Returns: |
| 630 | RuleDetail or None if not found |
| 631 | """ |
| 632 | await rules_cache.ensure_loaded() |
| 633 | |
| 634 | rule = rules_cache.get_rule_by_name(rule_name) |
| 635 | |
| 636 | if rule is None: |
| 637 | return None |
| 638 | |
| 639 | return rule_to_detail(rule) |
| 640 | |
| 641 | |
| 642 | async def get_rules_stats() -> dict: |
| 643 | """ |
| 644 | Get statistics about loaded detection rules. |
| 645 | |
| 646 | Returns: |
| 647 | Dictionary with rule statistics |
| 648 | """ |
| 649 | await rules_cache.ensure_loaded() |
| 650 | |
| 651 | stats = rules_cache.get_stats() |
| 652 | |
| 653 | return { |
| 654 | "total_rules": stats["total_rules"], |
| 655 | "by_platform": stats["by_platform"], |
| 656 | "by_status": stats["by_status"], |
| 657 | "by_severity": stats["by_severity"], |
| 658 | "by_mitre_tactic": stats["by_mitre_tactic"], |
| 659 | "rules_with_graylog": stats["rules_with_graylog"], |
| 660 | "last_refreshed": rules_cache.last_refresh, |
| 661 | "cache_ttl_minutes": CACHE_TTL_MINUTES, |
| 662 | } |
| 663 | |
| 664 | |
| 665 | async def refresh_rules_cache() -> dict: |
| 666 | """ |
| 667 | Manually refresh the rules cache from GitHub. |
| 668 | |
| 669 | Returns: |
| 670 | Dictionary with refresh results |
| 671 | """ |
| 672 | rules_loaded = await rules_cache.refresh() |
| 673 | |
| 674 | return { |
| 675 | "success": True, |
| 676 | "message": "Rules cache refreshed successfully", |
| 677 | "rules_loaded": rules_loaded, |
| 678 | "timestamp": datetime.utcnow(), |
| 679 | } |
| 680 | |
| 681 | |
| 682 | async def get_cache_health() -> dict: |
| 683 | """ |
| 684 | Get health/status information about the rules cache. |
| 685 | |
| 686 | Returns: |
| 687 | Dictionary with cache health information |
| 688 | """ |
| 689 | return { |
| 690 | "status": "healthy" if not rules_cache.is_stale else "stale", |
| 691 | "rules_loaded": rules_cache.rules_count, |
| 692 | "cache_age_minutes": rules_cache.cache_age_minutes, |
| 693 | "github_repo": GITHUB_REPO, |
| 694 | } |
| 695 | |
| 696 | |
| 697 | # ============================================================================= |
| 698 | # Search Execution Functions |
| 699 | # ============================================================================= |
| 700 | |
| 701 | |
| 702 | def _substitute_parameters(obj: Any, parameters: dict[str, Any]) -> Any: |
| 703 | """ |
| 704 | Recursively substitute ${PARAM_NAME} placeholders in a query object. |
| 705 | |
| 706 | Args: |
| 707 | obj: The object to substitute parameters in (dict, list, or str) |
| 708 | parameters: Dictionary of parameter names to values |
| 709 | |
| 710 | Returns: |
| 711 | The object with parameters substituted |
| 712 | """ |
| 713 | if isinstance(obj, str): |
| 714 | # Find all ${PARAM_NAME} patterns and replace them |
| 715 | pattern = r"\$\{([^}]+)\}" |
| 716 | |
| 717 | def replacer(match): |
| 718 | param_name = match.group(1) |
| 719 | if param_name in parameters: |
| 720 | value = parameters[param_name] |
| 721 | # If the entire string is just the placeholder, return the value directly |
| 722 | # This preserves types (int, bool, etc.) |
| 723 | if match.group(0) == obj: |
| 724 | return value |
| 725 | # Otherwise, convert to string for embedding |
| 726 | return str(value) |
| 727 | # Return original if parameter not found |
| 728 | return match.group(0) |
| 729 | |
| 730 | # Check if entire string is a single placeholder |
| 731 | full_match = re.fullmatch(pattern, obj) |
| 732 | if full_match: |
| 733 | param_name = full_match.group(1) |
| 734 | if param_name in parameters: |
| 735 | return parameters[param_name] |
| 736 | |
| 737 | # Otherwise do string substitution |
| 738 | return re.sub(pattern, replacer, obj) |
| 739 | |
| 740 | elif isinstance(obj, dict): |
| 741 | return {key: _substitute_parameters(value, parameters) for key, value in obj.items()} |
| 742 | |
| 743 | elif isinstance(obj, list): |
| 744 | return [_substitute_parameters(item, parameters) for item in obj] |
| 745 | |
| 746 | else: |
| 747 | return obj |
| 748 | |
| 749 | |
| 750 | def _validate_parameters( |
| 751 | rule: dict, |
| 752 | provided_params: dict[str, Any], |
| 753 | ) -> tuple[dict[str, Any], list[SearchValidationError]]: |
| 754 | """ |
| 755 | Validate and merge provided parameters with defaults. |
| 756 | |
| 757 | Args: |
| 758 | rule: The rule definition |
| 759 | provided_params: Parameters provided by the user |
| 760 | |
| 761 | Returns: |
| 762 | Tuple of (merged_params, validation_errors) |
| 763 | """ |
| 764 | errors: list[SearchValidationError] = [] |
| 765 | merged_params: dict[str, Any] = {} |
| 766 | |
| 767 | rule_params = rule.get("parameters", {}) |
| 768 | |
| 769 | for param_name, param_def in rule_params.items(): |
| 770 | is_required = param_def.get("required", False) |
| 771 | default_value = param_def.get("default") |
| 772 | |
| 773 | if param_name in provided_params: |
| 774 | # User provided the parameter |
| 775 | merged_params[param_name] = provided_params[param_name] |
| 776 | elif default_value is not None: |
| 777 | # Use default value |
| 778 | merged_params[param_name] = default_value |
| 779 | elif is_required: |
| 780 | # Required parameter missing |
| 781 | errors.append( |
| 782 | SearchValidationError( |
| 783 | parameter=param_name, |
| 784 | message=f"Required parameter '{param_name}' is missing. {param_def.get('description', '')}", |
| 785 | ), |
| 786 | ) |
| 787 | |
| 788 | return merged_params, errors |
| 789 | |
| 790 | |
| 791 | async def execute_rule_search( |
| 792 | request: ExecuteSearchRequest, |
| 793 | ) -> ExecuteSearchResponse: |
| 794 | """ |
| 795 | Execute a search against the Wazuh indexer using a rule definition. |
| 796 | |
| 797 | Args: |
| 798 | request: The search execution request |
| 799 | |
| 800 | Returns: |
| 801 | ExecuteSearchResponse with search results |
| 802 | |
| 803 | Raises: |
| 804 | ValueError: If the rule is not found or validation fails |
| 805 | """ |
| 806 | await rules_cache.ensure_loaded() |
| 807 | |
| 808 | # Get the rule |
| 809 | rule = rules_cache.get_rule_by_id(request.rule_id) |
| 810 | if rule is None: |
| 811 | raise ValueError(f"Rule with ID '{request.rule_id}' not found") |
| 812 | |
| 813 | # Add INDEX_PATTERN to provided parameters |
| 814 | all_params = {**request.parameters, "INDEX_PATTERN": request.index_pattern} |
| 815 | |
| 816 | # Validate parameters |
| 817 | merged_params, validation_errors = _validate_parameters(rule, all_params) |
| 818 | |
| 819 | if validation_errors: |
| 820 | error_messages = [f"{e.parameter}: {e.message}" for e in validation_errors] |
| 821 | raise ValueError(f"Parameter validation failed: {'; '.join(error_messages)}") |
| 822 | |
| 823 | # Get the search definition from the rule |
| 824 | search_def = rule.get("search", {}) |
| 825 | if not search_def: |
| 826 | raise ValueError(f"Rule '{request.rule_id}' does not contain a search definition") |
| 827 | |
| 828 | # Build the query with parameter substitution |
| 829 | query = search_def.get("query", {}) |
| 830 | substituted_query = _substitute_parameters(copy.deepcopy(query), merged_params) |
| 831 | |
| 832 | # Build the full search body |
| 833 | search_body: dict[str, Any] = { |
| 834 | "query": substituted_query, |
| 835 | } |
| 836 | |
| 837 | # Add size (from request override, rule definition, or default) |
| 838 | if request.size is not None: |
| 839 | search_body["size"] = request.size |
| 840 | elif "size" in search_def: |
| 841 | search_body["size"] = search_def["size"] |
| 842 | else: |
| 843 | search_body["size"] = 100 |
| 844 | |
| 845 | # Add sort if defined |
| 846 | if "sort" in search_def: |
| 847 | search_body["sort"] = _substitute_parameters( |
| 848 | copy.deepcopy(search_def["sort"]), |
| 849 | merged_params, |
| 850 | ) |
| 851 | |
| 852 | # Add _source if defined |
| 853 | if "_source" in search_def: |
| 854 | search_body["_source"] = search_def["_source"] |
| 855 | |
| 856 | logger.info(f"Executing search for rule '{request.rule_id}' on index '{request.index_pattern}'") |
| 857 | logger.debug(f"Search body: {json.dumps(search_body, indent=2)}") |
| 858 | |
| 859 | # Create the async Elasticsearch client |
| 860 | es_client = await create_wazuh_indexer_client_async() |
| 861 | |
| 862 | try: |
| 863 | # Execute the search |
| 864 | response = await es_client.search( |
| 865 | index=request.index_pattern, |
| 866 | body=search_body, |
| 867 | ) |
| 868 | |
| 869 | # Parse the response |
| 870 | hits_data = response.get("hits", {}) |
| 871 | total_hits = hits_data.get("total", {}) |
| 872 | if isinstance(total_hits, dict): |
| 873 | total_count = total_hits.get("value", 0) |
| 874 | else: |
| 875 | total_count = total_hits |
| 876 | |
| 877 | hits = [] |
| 878 | for hit in hits_data.get("hits", []): |
| 879 | hits.append( |
| 880 | SearchHit( |
| 881 | index=hit.get("_index", ""), |
| 882 | id=hit.get("_id", ""), |
| 883 | score=hit.get("_score"), |
| 884 | source=hit.get("_source", {}), |
| 885 | ), |
| 886 | ) |
| 887 | |
| 888 | return ExecuteSearchResponse( |
| 889 | success=True, |
| 890 | message="Search executed successfully", |
| 891 | rule_id=request.rule_id, |
| 892 | rule_name=rule.get("name", ""), |
| 893 | total_hits=total_count, |
| 894 | returned_hits=len(hits), |
| 895 | took_ms=response.get("took", 0), |
| 896 | hits=hits, |
| 897 | query_executed=search_body, |
| 898 | ) |
| 899 | |
| 900 | except Exception as e: |
| 901 | logger.error(f"Search execution failed: {e}") |
| 902 | raise ValueError(f"Search execution failed: {str(e)}") |
| 903 | |
| 904 | finally: |
| 905 | # Close the client |
| 906 | await es_client.close() |
| 907 | |
| 908 | |
| 909 | # ============================================================================= |
| 910 | # Graylog Query Functions |
| 911 | # ============================================================================= |
| 912 | |
| 913 | |
| 914 | async def generate_graylog_query( |
| 915 | request: ExecuteGraylogQueryRequest, |
| 916 | ) -> GraylogQueryResponse: |
| 917 | """ |
| 918 | Generate a Graylog query string from a rule with parameter substitution. |
| 919 | |
| 920 | Args: |
| 921 | request: The Graylog query request |
| 922 | |
| 923 | Returns: |
| 924 | GraylogQueryResponse with the substituted query |
| 925 | |
| 926 | Raises: |
| 927 | ValueError: If the rule is not found or has no Graylog query |
| 928 | """ |
| 929 | await rules_cache.ensure_loaded() |
| 930 | |
| 931 | # Get the rule |
| 932 | rule = rules_cache.get_rule_by_id(request.rule_id) |
| 933 | if rule is None: |
| 934 | raise ValueError(f"Rule with ID '{request.rule_id}' not found") |
| 935 | |
| 936 | # Check if rule has Graylog query |
| 937 | graylog_data = rule.get("graylog") |
| 938 | if not graylog_data or not isinstance(graylog_data, dict): |
| 939 | raise ValueError(f"Rule '{request.rule_id}' does not contain a Graylog query") |
| 940 | |
| 941 | original_query = graylog_data.get("query", "") |
| 942 | if not original_query: |
| 943 | raise ValueError(f"Rule '{request.rule_id}' has an empty Graylog query") |
| 944 | |
| 945 | # Validate parameters |
| 946 | merged_params, validation_errors = _validate_parameters(rule, request.parameters) |
| 947 | |
| 948 | if validation_errors: |
| 949 | error_messages = [f"{e.parameter}: {e.message}" for e in validation_errors] |
| 950 | raise ValueError(f"Parameter validation failed: {'; '.join(error_messages)}") |
| 951 | |
| 952 | # Substitute parameters in the Graylog query |
| 953 | substituted_query = _substitute_parameters(original_query, merged_params) |
| 954 | |
| 955 | logger.info(f"Generated Graylog query for rule '{request.rule_id}'") |
| 956 | |
| 957 | return GraylogQueryResponse( |
| 958 | success=True, |
| 959 | message="Graylog query generated successfully", |
| 960 | rule_id=request.rule_id, |
| 961 | rule_name=rule.get("name", ""), |
| 962 | graylog_query=substituted_query, |
| 963 | original_query=original_query, |
| 964 | ) |
| 965 | |
| 966 | |
| 967 | # ============================================================================= |
| 968 | # Graylog Alert Provisioning Functions |
| 969 | # ============================================================================= |
| 970 | |
| 971 | |
| 972 | async def provision_graylog_alert_from_rule( |
| 973 | request: ProvisionGraylogAlertRequest, |
| 974 | ) -> ProvisionGraylogAlertResponse: |
| 975 | """ |
| 976 | Provision a Graylog event definition from a CoPilot Search rule. |
| 977 | |
| 978 | This takes a rule with a Graylog query and creates a Graylog event definition |
| 979 | that will alert when the query matches. |
| 980 | |
| 981 | Args: |
| 982 | request: The provisioning request |
| 983 | |
| 984 | Returns: |
| 985 | ProvisionGraylogAlertResponse with the result |
| 986 | |
| 987 | Raises: |
| 988 | ValueError: If the rule is not found or has no Graylog query |
| 989 | """ |
| 990 | await rules_cache.ensure_loaded() |
| 991 | |
| 992 | # Get the rule |
| 993 | rule = rules_cache.get_rule_by_id(request.rule_id) |
| 994 | if rule is None: |
| 995 | raise ValueError(f"Rule with ID '{request.rule_id}' not found") |
| 996 | |
| 997 | # Check if rule has Graylog query |
| 998 | graylog_data = rule.get("graylog") |
| 999 | if not graylog_data or not isinstance(graylog_data, dict): |
| 1000 | raise ValueError(f"Rule '{request.rule_id}' does not contain a Graylog query") |
| 1001 | |
| 1002 | graylog_query = graylog_data.get("query", "") |
| 1003 | if not graylog_query: |
| 1004 | raise ValueError(f"Rule '{request.rule_id}' has an empty Graylog query") |
| 1005 | |
| 1006 | # Get rule metadata |
| 1007 | rule_name = rule.get("name", request.rule_id) |
| 1008 | rule_description = rule.get("description", "") |
| 1009 | rule_severity = rule.get("response", {}).get("severity", "medium") |
| 1010 | alert_source = _get_alert_source_from_rule(rule) |
| 1011 | |
| 1012 | # Determine the alert title |
| 1013 | alert_title = request.custom_title if request.custom_title else rule_name.upper().replace(" ", " - ") |
| 1014 | |
| 1015 | # Determine priority from rule severity or request |
| 1016 | priority = request.priority if request.priority != 2 else _get_priority_from_severity(rule_severity) |
| 1017 | |
| 1018 | logger.info(f"Provisioning Graylog alert for rule '{request.rule_id}' with title '{alert_title}'") |
| 1019 | |
| 1020 | # Build the Graylog event definition model |
| 1021 | alert_model = GraylogAlertProvisionModel( |
| 1022 | title=alert_title, |
| 1023 | description=rule_description, |
| 1024 | priority=priority, |
| 1025 | config=GraylogAlertProvisionConfig( |
| 1026 | type="aggregation-v1", |
| 1027 | query=graylog_query, |
| 1028 | query_parameters=[], |
| 1029 | streams=request.streams, |
| 1030 | group_by=[], |
| 1031 | series=[], |
| 1032 | conditions={ |
| 1033 | "expression": None, |
| 1034 | }, |
| 1035 | search_within_ms=_convert_seconds_to_milliseconds(request.search_within_seconds), |
| 1036 | execute_every_ms=_convert_seconds_to_milliseconds(request.execute_every_seconds), |
| 1037 | event_limit=request.event_limit, |
| 1038 | ), |
| 1039 | field_spec={ |
| 1040 | "ALERT_ID": GraylogAlertProvisionFieldSpecItem( |
| 1041 | data_type="string", |
| 1042 | providers=[ |
| 1043 | GraylogAlertProvisionProvider( |
| 1044 | type="template-v1", |
| 1045 | template="${source._id}", |
| 1046 | require_values=True, |
| 1047 | ), |
| 1048 | ], |
| 1049 | ), |
| 1050 | "CUSTOMER_CODE": GraylogAlertProvisionFieldSpecItem( |
| 1051 | data_type="string", |
| 1052 | providers=[ |
| 1053 | GraylogAlertProvisionProvider( |
| 1054 | type="template-v1", |
| 1055 | template="${source.agent_labels_customer}", |
| 1056 | require_values=True, |
| 1057 | ), |
| 1058 | ], |
| 1059 | ), |
| 1060 | "ALERT_SOURCE": GraylogAlertProvisionFieldSpecItem( |
| 1061 | data_type="string", |
| 1062 | providers=[ |
| 1063 | GraylogAlertProvisionProvider( |
| 1064 | type="template-v1", |
| 1065 | template=alert_source, |
| 1066 | require_values=True, |
| 1067 | ), |
| 1068 | ], |
| 1069 | ), |
| 1070 | "COPILOT_ALERT_ID": GraylogAlertProvisionFieldSpecItem( |
| 1071 | data_type="string", |
| 1072 | providers=[ |
| 1073 | GraylogAlertProvisionProvider( |
| 1074 | type="template-v1", |
| 1075 | template="NONE", |
| 1076 | require_values=True, |
| 1077 | ), |
| 1078 | ], |
| 1079 | ), |
| 1080 | "RULE_ID": GraylogAlertProvisionFieldSpecItem( |
| 1081 | data_type="string", |
| 1082 | providers=[ |
| 1083 | GraylogAlertProvisionProvider( |
| 1084 | type="template-v1", |
| 1085 | template=request.rule_id, |
| 1086 | require_values=True, |
| 1087 | ), |
| 1088 | ], |
| 1089 | ), |
| 1090 | }, |
| 1091 | key_spec=[], |
| 1092 | notification_settings=GraylogAlertProvisionNotificationSettings( |
| 1093 | grace_period_ms=0, |
| 1094 | backlog_size=None, |
| 1095 | ), |
| 1096 | alert=True, |
| 1097 | ) |
| 1098 | |
| 1099 | # Provision the alert definition |
| 1100 | await provision_alert_definition(alert_model) |
| 1101 | |
| 1102 | logger.info(f"Successfully provisioned Graylog alert '{alert_title}' for rule '{request.rule_id}'") |
| 1103 | |
| 1104 | return ProvisionGraylogAlertResponse( |
| 1105 | success=True, |
| 1106 | message=f"Graylog alert '{alert_title}' provisioned successfully", |
| 1107 | rule_id=request.rule_id, |
| 1108 | rule_name=rule_name, |
| 1109 | alert_title=alert_title, |
| 1110 | graylog_query=graylog_query, |
| 1111 | ) |