@cryptotaxi247 / CoPilot / commits / 43c4a80b

Office365 manual alert (#213)

* escalate alert logic WIP * manual alert escalation rewrite

taylor_socfortress committed May 14, 2024 at 07:53 UTC 43c4a80b351047d9cd43311c9e52799d382e7827
4 files changed +622 -3
backend/app/integrations/alert_escalation/routes/escalate_alert.py new
+37
@@ -0,0 +1,37 @@
1 +from fastapi import APIRouter
2 +from fastapi import Depends
3 +from fastapi import Security
4 +from loguru import logger
5 +from sqlalchemy.ext.asyncio import AsyncSession
6 +
7 +from app.auth.utils import AuthHandler
8 +from app.db.db_session import get_db
9 +from app.integrations.alert_escalation.schema.escalate_alert import CreateAlertRequest
10 +from app.integrations.alert_escalation.schema.escalate_alert import CreateAlertResponse
11 +from app.integrations.alert_escalation.services.escalate_alert import create_alert
12 +
13 +integration_escalate_alerts_router = APIRouter()
14 +
15 +
16 +@integration_escalate_alerts_router.post(
17 + "/create",
18 + response_model=CreateAlertResponse,
19 + description="Manually create an alert in IRIS from Copilot WebUI",
20 + dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
21 +)
22 +async def create_alert_route(
23 + create_alert_request: CreateAlertRequest,
24 + session: AsyncSession = Depends(get_db),
25 +) -> CreateAlertResponse:
26 + """
27 + Create an alert in IRIS. Manually create an alert in IRIS from Copilot WebUI.
28 +
29 + Args:
30 + create_alert_request (CreateAlertRequest): The request object containing the details of the alert to be created.
31 + session (AsyncSession, optional): The database session. Defaults to Depends(get_session).
32 +
33 + Returns:
34 + CreateAlertResponse: The response object containing the result of the alert creation.
35 + """
36 + logger.info(f"Creating alert {create_alert_request.alert_id} in IRIS")
37 + return await create_alert(create_alert_request, session)
backend/app/integrations/alert_escalation/schema/escalate_alert.py new
+208
@@ -0,0 +1,208 @@
1 +from enum import Enum
2 +from typing import Any
3 +from typing import Dict
4 +from typing import Optional
5 +
6 +from pydantic import BaseModel
7 +from pydantic import Extra
8 +from pydantic import Field
9 +
10 +
11 +class CustomerCodeKeys(Enum):
12 + AGENT_LABELS_CUSTOMER = "agent_labels_customer"
13 + DATA_OFFICE365_ORGANIZATION_ID = "data_office365_OrganizationId"
14 + SYSLOG_CUSTOMER = "syslog_customer"
15 +
16 +
17 +class SyslogLevelMapping(Enum):
18 + INFO = 1
19 + NOTICE = 2
20 + WARNING = 3
21 + ALERT = 4
22 +
23 +
24 +class SourceFieldsToRemove(Enum):
25 + GL2 = "gl2"
26 + # Add more fields as needed
27 +
28 +
29 +class ValidIocFields(Enum):
30 + MISP_VALUE = "misp_value"
31 + OPENCTI_VALUE = "opencti_value"
32 + THREAT_INTEL_VALUE = "threat_intel_value"
33 +
34 +
35 +class CreateAlertRequest(BaseModel):
36 + index_name: str = Field(
37 + ...,
38 + description="The name of the index to search alerts for.",
39 + )
40 + alert_id: str = Field(..., description="The alert id.")
41 +
42 +
43 +class CreateAlertResponse(BaseModel):
44 + success: bool
45 + message: str
46 + alert_id: int = Field(..., description="The alert id as created in IRIS.")
47 + alert_url: str = Field(..., description="The alert url as created in IRIS.")
48 +
49 +
50 +class GenericSourceModel(BaseModel):
51 + timestamp: str = Field(..., description="The timestamp of the alert.")
52 + timestamp_utc: Optional[str] = Field(
53 + ...,
54 + description="The UTC timestamp of the alert.",
55 + )
56 + rule_description: Optional[str] = Field(
57 + "No autogenerated rule_description found",
58 + description="The timefield of the alert to be used when creating the IRIS alert.",
59 + )
60 + syslog_level: Optional[str] = Field(
61 + "No autogenerated syslog_level found",
62 + description="The timefield of the alert to be used when creating the IRIS alert.",
63 + )
64 +
65 + class Config:
66 + extra = Extra.allow
67 +
68 + def to_dict(self):
69 + return self.dict(exclude_none=True)
70 +
71 +
72 +class GenericAlertModel(BaseModel):
73 + _index: str
74 + _id: str
75 + _version: int
76 + _source: GenericSourceModel # Nested model
77 + asset_type_id: Optional[int] = Field(
78 + None,
79 + description="The asset type id of the alert which is needed for when we add the asset to IRIS.",
80 + )
81 + ioc_value: Optional[str] = Field(
82 + None,
83 + description="The IoC value of the alert which is needed for when we add the IoC to IRIS.",
84 + )
85 + ioc_type: Optional[str] = Field(
86 + None,
87 + description="The IoC type of the alert which is needed for when we add the IoC to IRIS.",
88 + )
89 + time_field: Optional[str] = Field(
90 + "timestamp",
91 + description="The timefield of the alert to be used when creating the IRIS alert.",
92 + )
93 + rule_description: Optional[str] = Field(
94 + "No autogenerated rule_description found",
95 + description="The timefield of the alert to be used when creating the IRIS alert.",
96 + )
97 + syslog_level: Optional[str] = Field(
98 + "No autogenerated syslog_level found",
99 + description="The timefield of the alert to be used when creating the IRIS alert.",
100 + )
101 +
102 + class Config:
103 + extra = Extra.allow
104 +
105 +
106 +# Sample data from `get_single_alert_details`
107 +sample_data = {
108 + "_index": "some_index",
109 + "_id": "some_id",
110 + "_version": 1,
111 + "_source": {
112 + "agent_name": "some_agent_name",
113 + "agent_id": "some_agent_id",
114 + # ... other fields
115 + },
116 + # ... other fields
117 +}
118 +
119 +
120 +########### Create Alerts Schemas ###########
121 +class IrisAsset(BaseModel):
122 + asset_name: str = Field(..., description="Name of the asset", example="Server01")
123 + asset_ip: str = Field(
124 + ...,
125 + description="IP address of the asset",
126 + example="192.168.1.1",
127 + )
128 + asset_description: str = Field(
129 + ...,
130 + description="Description of the asset",
131 + example="Windows Server",
132 + )
133 + asset_type_id: int = Field(..., description="Type ID of the asset", example=1)
134 + asset_tags: Optional[str] = Field(
135 + "Agent ID not found. Ensure the agent has been registered with Wazuh Manager and synced to the Agents table.",
136 + description="Tags of the asset",
137 + example="001",
138 + )
139 +
140 + def to_dict(self):
141 + return self.dict(exclude_none=True)
142 +
143 +
144 +class IrisIoc(BaseModel):
145 + ioc_value: str = Field(
146 + ...,
147 + description="Value of the IoC",
148 + example="www.google.com",
149 + )
150 + ioc_description: str = Field(
151 + ...,
152 + description="Description of the IoC",
153 + example="Google",
154 + )
155 + ioc_tlp_id: int = Field(1, description="TLP ID of the IoC", example=1)
156 + ioc_type_id: int = Field(20, description="Type ID of the IoC", example=20)
157 +
158 + def to_dict(self):
159 + return self.dict(exclude_none=True)
160 +
161 +
162 +class IrisAlertContext(BaseModel):
163 + alert_id: str = Field(..., description="ID of the alert", example="123")
164 + alert_name: str = Field(
165 + ...,
166 + description="Name of the alert",
167 + example="Intrusion Detected",
168 + )
169 + alert_level: int = Field(..., description="Severity level of the alert", example=3)
170 +
171 + class Config:
172 + extra = Extra.allow
173 +
174 +
175 +class IrisAlertPayload(BaseModel):
176 + alert_title: str = Field(
177 + ...,
178 + description="Title of the alert",
179 + example="Intrusion Detected",
180 + )
181 + alert_description: str = Field(
182 + ...,
183 + description="Description of the alert",
184 + example="Intrusion Detected by Firewall",
185 + )
186 + alert_source: str = Field(..., description="Source of the alert", example="Wazuh")
187 + alert_status_id: int = Field(..., description="Status ID of the alert", example=3)
188 + alert_severity_id: int = Field(
189 + ...,
190 + description="Severity ID of the alert",
191 + example=5,
192 + )
193 + alert_customer_id: int = Field(
194 + ...,
195 + description="Customer ID related to the alert",
196 + example=1,
197 + )
198 + alert_source_content: Dict[str, Any] = Field(
199 + ...,
200 + description="Original content from the alert source",
201 + )
202 + alert_context: IrisAlertContext = Field(
203 + ...,
204 + description="Contextual information about the alert",
205 + )
206 +
207 + def to_dict(self):
208 + return self.dict(exclude_none=True)
backend/app/integrations/alert_escalation/services/escalate_alert.py new
+374
@@ -0,0 +1,374 @@
1 +from typing import Optional
2 +
3 +from fastapi import HTTPException
4 +from loguru import logger
5 +from sqlalchemy.ext.asyncio import AsyncSession
6 +from sqlalchemy.future import select
7 +
8 +from app.connectors.dfir_iris.utils.universal import fetch_and_validate_data
9 +from app.connectors.dfir_iris.utils.universal import initialize_client_and_alert
10 +from app.connectors.utils import get_connector_info_from_db
11 +from app.connectors.wazuh_indexer.utils.universal import create_wazuh_indexer_client
12 +from app.integrations.alert_creation_settings.models.alert_creation_settings import (
13 + AlertCreationSettings,
14 +)
15 +from app.integrations.alert_escalation.schema.escalate_alert import CreateAlertRequest
16 +from app.integrations.alert_escalation.schema.escalate_alert import CreateAlertResponse
17 +from app.integrations.alert_escalation.schema.escalate_alert import CustomerCodeKeys
18 +from app.integrations.alert_escalation.schema.escalate_alert import GenericAlertModel
19 +from app.integrations.alert_escalation.schema.escalate_alert import GenericSourceModel
20 +from app.integrations.alert_escalation.schema.escalate_alert import IrisAlertContext
21 +from app.integrations.alert_escalation.schema.escalate_alert import IrisAlertPayload
22 +from app.integrations.alert_escalation.schema.escalate_alert import SourceFieldsToRemove
23 +from app.integrations.alert_escalation.schema.escalate_alert import SyslogLevelMapping
24 +
25 +
26 +async def fetch_settings(field: str, value: str, session: AsyncSession):
27 + """
28 + Fetch settings based on the field and value.
29 +
30 + Args:
31 + field (str): The field to check.
32 + value (str): The value to check.
33 + session (AsyncSession): The database session.
34 +
35 + Returns:
36 + AlertCreationSettings: The settings if found, None otherwise.
37 + """
38 + logger.info(f"Checking if {field}: {value} is valid.")
39 + result = await session.execute(
40 + select(AlertCreationSettings).where(
41 + getattr(AlertCreationSettings, field) == value,
42 + ),
43 + )
44 + settings = result.scalars().first()
45 + logger.info(f"Settings: {settings}")
46 + return settings
47 +
48 +
49 +async def is_customer_code_valid(customer_code: str, session: AsyncSession) -> AlertCreationSettings:
50 + """
51 + Check if the customer code is valid.
52 +
53 + Args:
54 + customer_code (str): The customer code to check.
55 + session (AsyncSession): The database session.
56 +
57 + Returns:
58 + bool: True if the customer code is valid, False otherwise.
59 + """
60 + settings = await fetch_settings("customer_code", customer_code, session)
61 +
62 + if settings:
63 + return settings
64 +
65 + # If no settings found with customer_code, try with office365_organization_id
66 + settings = await fetch_settings("office365_organization_id", customer_code, session)
67 +
68 + if settings:
69 + return settings
70 +
71 + raise HTTPException(
72 + status_code=400,
73 + detail=f"Customer code {customer_code} is not valid. Has the customer been provisioned?",
74 + )
75 +
76 +
77 +async def get_single_alert_details(
78 + alert_details: CreateAlertRequest,
79 +) -> GenericAlertModel:
80 + """
81 + Fetches the details of a single alert.
82 +
83 + Args:
84 + alert_details (CreateAlertRequest): The details of the alert to fetch.
85 +
86 + Returns:
87 + GenericAlertModel: The model representing the fetched alert.
88 +
89 + Raises:
90 + HTTPException: If there is an error while fetching the alert details.
91 + """
92 + logger.info(
93 + f"Fetching alert details for alert {alert_details.alert_id} in index {alert_details.index_name}",
94 + )
95 + es_client = await create_wazuh_indexer_client("Wazuh-Indexer")
96 + try:
97 + alert = es_client.get(index=alert_details.index_name, id=alert_details.alert_id)
98 + source_model = GenericSourceModel(**alert["_source"])
99 + return GenericAlertModel(
100 + _source=source_model,
101 + _id=alert["_id"],
102 + _index=alert["_index"],
103 + _version=alert["_version"],
104 + rule_description=source_model.rule_description,
105 + syslog_level=source_model.syslog_level,
106 + )
107 + except Exception as e:
108 + logger.debug(f"Failed to collect alert details: {e}")
109 + raise HTTPException(
110 + status_code=400,
111 + detail=f"Failed to collect alert details: {e}",
112 + )
113 +
114 +
115 +async def set_alert_level(syslog_level: str):
116 + """
117 + Sets the alert level based on the syslog level.
118 +
119 + Args:
120 + syslog_level (str): The syslog level.
121 +
122 + Returns:
123 + int: The alert level.
124 + """
125 + for level in SyslogLevelMapping:
126 + if level.name == syslog_level:
127 + logger.info(f"Setting alert level to {level.value}")
128 + return level.value
129 + return 3
130 +
131 +
132 +async def build_alert_context_payload(
133 + alert_details: GenericAlertModel,
134 + customer_alert_creation_settings: AlertCreationSettings,
135 +) -> IrisAlertContext:
136 + """
137 + Builds the payload for the alert context.
138 +
139 + Args:
140 + alert_details (GenericAlertModel): The details of the alert.
141 + agent_data (AgentsResponse): The data of the agent.
142 + session (AsyncSession): The async session.
143 +
144 + Returns:
145 + IrisAlertContext: The built alert context payload.
146 + """
147 + # Convert the _source to a dictionary
148 + source_dict = alert_details._source.to_dict()
149 +
150 + # Remove fields that start with any prefix in SourceFieldsToRemove
151 + for field in SourceFieldsToRemove:
152 + source_dict = {k: v for k, v in source_dict.items() if not k.startswith(field.value)}
153 +
154 + return IrisAlertContext(
155 + customer_iris_id=customer_alert_creation_settings.iris_customer_id,
156 + customer_name=customer_alert_creation_settings.customer_name,
157 + customer_cases_index=customer_alert_creation_settings.iris_index,
158 + alert_id=alert_details._id,
159 + alert_name=alert_details.rule_description,
160 + alert_level=await set_alert_level(alert_details.syslog_level),
161 + **source_dict,
162 + )
163 +
164 +
165 +async def construct_soc_alert_url(root_url: str, soc_alert_id: int) -> str:
166 + """Constructs the full URL for the SOC alert.
167 +
168 + Args:
169 + root_url (str): The root URL of the SOC alert system.
170 + soc_alert_id (int): The ID of the SOC alert.
171 +
172 + Returns:
173 + str: The full URL for the SOC alert.
174 +
175 + """
176 + url_path = f"/alerts?cid=1&page=1&per_page=10&sort=desc&alert_ids={soc_alert_id}"
177 + return f"{root_url}{url_path}"
178 +
179 +
180 +async def add_alert_to_document(
181 + es_client,
182 + alert: CreateAlertRequest,
183 + soc_alert_id: int,
184 + session: AsyncSession,
185 +) -> Optional[str]:
186 + """
187 + Update the alert document in Elasticsearch with the provided SOC alert ID URL.
188 +
189 + Parameters:
190 + - es_client: The Elasticsearch client instance to use for the update.
191 + - alert: The alert request object containing alert_id and index_name.
192 + - soc_alert_id: The alert ID as it exists within IRIS.
193 + - session: The database session for retrieving connector information.
194 +
195 + Returns:
196 + - True if the update is successful, False otherwise.
197 + """
198 + try:
199 + connector_info = await get_connector_info_from_db("DFIR-IRIS", session)
200 + full_url = await construct_soc_alert_url(
201 + connector_info["connector_url"],
202 + soc_alert_id,
203 + )
204 + es_client.update(
205 + index=alert.index_name,
206 + id=alert.alert_id,
207 + body={"doc": {"alert_url": full_url}},
208 + )
209 + logger.info(
210 + f"Added alert ID {soc_alert_id} to alert {alert.alert_id} in index {alert.index_name}",
211 + )
212 + return full_url
213 + except Exception as e:
214 + logger.error(
215 + f"Failed to add alert ID {soc_alert_id} to alert {alert.alert_id} in index {alert.index_name}: {e}",
216 + )
217 + # Attempt to remove read-only block
218 + try:
219 + es_client.indices.put_settings(
220 + index=alert.index_name,
221 + body={"index.blocks.write": None},
222 + )
223 + logger.info(
224 + f"Removed read-only block from index {alert.index_name}. Retrying update.",
225 + )
226 +
227 + # Retry the update operation
228 + es_client.update(
229 + index=alert.index_name,
230 + id=alert.alert_id,
231 + body={"doc": {"alert_url": full_url}},
232 + )
233 + logger.info(
234 + f"Added alert ID {soc_alert_id} to alert {alert.alert_id} in index {alert.index_name} after removing read-only block",
235 + )
236 +
237 + # Reenable the write block
238 + es_client.indices.put_settings(
239 + index=alert.index_name,
240 + body={"index.blocks.write": True},
241 + )
242 + return full_url
243 + except Exception as e2:
244 + logger.error(
245 + f"Failed to remove read-only block from index {alert.index_name}: {e2}",
246 + )
247 + return False
248 +
249 +
250 +async def get_customer_code(alert_details: dict):
251 + logger.info(f"Fetching customer code for alert {alert_details}")
252 +
253 + # Iterate over the possible keys and return the value if the key is present
254 + for key in CustomerCodeKeys:
255 + logger.info(f"Checking for key {key.value}")
256 + if key.value in alert_details:
257 + return alert_details[key.value]
258 +
259 + # If none of the keys are present, raise an exception
260 + logger.info(f"Failed to fetch customer code. Valid customer code field names are {', '.join([key.value for key in CustomerCodeKeys])}")
261 + raise HTTPException(
262 + status_code=400,
263 + detail=f"Failed to fetch customer code. Valid customer code field names are {', '.join([key.value for key in CustomerCodeKeys])}",
264 + )
265 +
266 +
267 +async def build_alert_payload(
268 + alert_details: GenericAlertModel,
269 + customer_alert_creation_settings: AlertCreationSettings,
270 +) -> IrisAlertPayload:
271 + """
272 + Builds the alert payload based on the provided alert details, agent data, IoC payload, and session.
273 +
274 + Args:
275 + alert_details (GenericAlertModel): The details of the alert.
276 + agent_data: The data of the agent.
277 + ioc_payload (Optional[IrisIoc]): The IoC payload.
278 + session (AsyncSession): The session object for database operations.
279 +
280 + Returns:
281 + IrisAlertPayload: The built alert payload.
282 +
283 + Raises:
284 + HTTPException: If there is an error while building the alert payload.
285 + """
286 + context_payload = await build_alert_context_payload(
287 + alert_details=alert_details,
288 + customer_alert_creation_settings=customer_alert_creation_settings,
289 + )
290 + logger.info(f"Context payload: {context_payload}")
291 + timefield = customer_alert_creation_settings.timefield
292 + # Get the timefield value from the alert_details
293 + if hasattr(alert_details, timefield):
294 + alert_details.time_field = getattr(alert_details, timefield)
295 + logger.info(f"Alert has context: {context_payload}")
296 + try:
297 + return IrisAlertPayload(
298 + alert_title=alert_details._source.rule_description,
299 + alert_description=alert_details._source.rule_description,
300 + alert_source="CoPilot - Manual Escalation",
301 + alert_status_id=3,
302 + alert_severity_id=5,
303 + alert_customer_id=customer_alert_creation_settings.iris_customer_id,
304 + alert_source_content=alert_details._source,
305 + alert_context=context_payload,
306 + alert_source_event_time=alert_details.time_field,
307 + )
308 + except Exception as e:
309 + logger.error(f"Failed to build alert payload: {e}")
310 + raise HTTPException(
311 + status_code=500,
312 + detail=f"Failed to build alert payload: {e}",
313 + )
314 +
315 +
316 +async def create_alert(
317 + alert: CreateAlertRequest,
318 + session: AsyncSession,
319 +) -> CreateAlertResponse:
320 + """
321 + Creates an alert in IRIS.
322 +
323 + Args:
324 + alert (CreateAlertRequest): The request object containing the alert details.
325 + session (AsyncSession): The database session.
326 +
327 + Returns:
328 + CreateAlertResponse: The response object containing the created alert details.
329 +
330 + Raises:
331 + HTTPException: If there is an error creating the alert.
332 + """
333 + logger.info(f"Creating alert {alert.alert_id} in IRIS")
334 + alert_details = await get_single_alert_details(alert_details=alert)
335 + logger.info(f"Alert details: {alert_details}")
336 +
337 + customer_code = await get_customer_code(dict(alert_details._source))
338 + logger.info(f"Customer code: {customer_code}")
339 + customer_alert_creation_settings = await is_customer_code_valid(customer_code=customer_code, session=session)
340 + logger.info(f"Customer creation settings: {customer_alert_creation_settings}")
341 + iris_alert_payload = await build_alert_payload(
342 + alert_details=alert_details,
343 + customer_alert_creation_settings=customer_alert_creation_settings,
344 + )
345 + logger.info(f"Iris Alert Payload: {iris_alert_payload}")
346 + client, alert_client = await initialize_client_and_alert("DFIR-IRIS")
347 + result = await fetch_and_validate_data(
348 + client,
349 + alert_client.add_alert,
350 + iris_alert_payload.to_dict(),
351 + )
352 + alert_id = result["data"]["alert_id"]
353 +
354 + es_client = await create_wazuh_indexer_client("Wazuh-Indexer")
355 + iris_url = await add_alert_to_document(
356 + es_client,
357 + alert,
358 + result["data"]["alert_id"],
359 + session=session,
360 + )
361 + try:
362 + alert_id = result["data"]["alert_id"]
363 + return CreateAlertResponse(
364 + alert_id=alert_id,
365 + success=True,
366 + message=f"Alert {alert_id} created successfully",
367 + alert_url=iris_url,
368 + )
369 + except Exception as e:
370 + logger.error(f"Failed to create alert {alert.alert_id}: {e}")
371 + raise HTTPException(
372 + status_code=500,
373 + detail=f"Failed to create alert for ID {alert.alert_id}: {e}",
374 + )
backend/app/routers/dfir_iris.py
+3 -3
@@ -5,8 +5,8 @@ from app.connectors.dfir_iris.routes.assets import dfir_iris_assets_router
5 from app.connectors.dfir_iris.routes.cases import dfir_iris_cases_router
6 from app.connectors.dfir_iris.routes.notes import dfir_iris_notes_router
7 from app.connectors.dfir_iris.routes.users import dfir_iris_users_router
8 -from app.integrations.alert_escalation.routes.general_alert import (
9 - integration_general_alerts_router,
8 +from app.integrations.alert_escalation.routes.escalate_alert import (
9 + integration_escalate_alerts_router,
10 )
11
12 # Instantiate the APIRouter
@@ -27,7 +27,7 @@ router.include_router(dfir_iris_cases_router, prefix="/soc/cases", tags=["soc-ca
27 router.include_router(dfir_iris_notes_router, prefix="/soc/notes", tags=["soc-notes"])
28 router.include_router(dfir_iris_users_router, prefix="/soc/users", tags=["soc-users"])
29 router.include_router(
30 - integration_general_alerts_router,
30 + integration_escalate_alerts_router,
31 prefix="/soc/general_alert",
32 tags=["soc-general-alerts"],
33 )