@cryptotaxi247 / CoPilot / commits / 3099e5ad

689 graylog network (#690)

* feat: add Graylog Network service and routing context for network logs * feat: implement Graylog-Network context handling and update related routes * precommit fixes

taylor_socfortress committed Feb 11, 2026 at 11:10 UTC 3099e5adf59c8c63a27fa4877d8890af4cd0a951
8 files changed +278 -112
.env.example
+4
@@ -36,6 +36,10 @@ GRAYLOG_URL=http://127.1.1.1
36 GRAYLOG_USERNAME=dummy
37 GRAYLOG_PASSWORD=dummy
38
39 +GRAYLOG_NETWORK_URL=http://127.1.1.1
40 +GRAYLOG_NETWORK_USERNAME=dummy
41 +GRAYLOG_NETWORK_PASSWORD=dummy
42 +
43 SHUFFLE_URL=https://127.1.1.1
44 SHUFFLER_API_KEY=dummy
45 SHUFFLE_WORKFLOW_ID=dummy
backend/app/connectors/graylog/utils/routing.py new
+98
@@ -0,0 +1,98 @@
1 +from contextvars import ContextVar
2 +from enum import Enum
3 +from typing import Optional
4 +
5 +from loguru import logger
6 +
7 +
8 +class GraylogContext(Enum):
9 + """
10 + Enum to define the context/purpose for Graylog connections.
11 +
12 + WAZUH: Default Graylog instance for Wazuh and third-party integrations (Graylog01)
13 + NETWORK: Graylog instance for network logs and syslog ingestion (Graylog02)
14 + """
15 +
16 + WAZUH = "Graylog"
17 + NETWORK = "Graylog-Network"
18 +
19 +
20 +# Context variable to hold the current Graylog context
21 +_graylog_context: ContextVar[Optional[GraylogContext]] = ContextVar(
22 + "graylog_context",
23 + default=None,
24 +)
25 +
26 +
27 +def set_graylog_context(context: GraylogContext) -> None:
28 + """
29 + Sets the Graylog context for the current async context.
30 + This should be called at the route/entry point level.
31 +
32 + Args:
33 + context (GraylogContext): The Graylog context to use for subsequent calls.
34 + """
35 + logger.debug(f"Setting Graylog context to: {context.value}")
36 + _graylog_context.set(context)
37 +
38 +
39 +def clear_graylog_context() -> None:
40 + """
41 + Clears the Graylog context, reverting to default behavior.
42 + """
43 + logger.debug("Clearing Graylog context")
44 + _graylog_context.set(None)
45 +
46 +
47 +def get_current_graylog_connector() -> str:
48 + """
49 + Gets the current Graylog connector name based on the context.
50 + Falls back to the default "Graylog" if no context is set.
51 +
52 + Returns:
53 + str: The connector name to use.
54 + """
55 + context = _graylog_context.get()
56 + if context is not None:
57 + return context.value
58 + return GraylogContext.WAZUH.value
59 +
60 +
61 +# Mapping of specific use cases to their Graylog context
62 +CONTEXT_MAPPING = {
63 + # Network-related integrations -> Graylog-Network
64 + "sonicwall": GraylogContext.NETWORK,
65 + "syslog": GraylogContext.NETWORK,
66 + "firewall": GraylogContext.NETWORK,
67 + "network": GraylogContext.NETWORK,
68 + # Wazuh/default integrations -> Graylog (default)
69 + "wazuh": GraylogContext.WAZUH,
70 + "default": GraylogContext.WAZUH,
71 +}
72 +
73 +
74 +def get_graylog_connector_name(
75 + context: Optional[GraylogContext] = None,
76 + use_case: Optional[str] = None,
77 +) -> str:
78 + """
79 + Returns the appropriate Graylog connector name based on context or use case.
80 + If neither is provided, checks for a context variable, then falls back to default.
81 +
82 + Args:
83 + context (Optional[GraylogContext]): The explicit context to use.
84 + use_case (Optional[str]): A string identifier for the use case.
85 +
86 + Returns:
87 + str: The connector name to use for database lookup.
88 + """
89 + if context is not None:
90 + return context.value
91 +
92 + if use_case is not None:
93 + use_case_lower = use_case.lower()
94 + mapped_context = CONTEXT_MAPPING.get(use_case_lower, GraylogContext.WAZUH)
95 + return mapped_context.value
96 +
97 + # Fall back to context variable or default
98 + return get_current_graylog_connector()
backend/app/connectors/graylog/utils/universal.py
+46 -27
@@ -6,6 +6,7 @@ import requests
6 from fastapi import HTTPException
7 from loguru import logger
8
9 +from app.connectors.graylog.utils.routing import get_current_graylog_connector
10 from app.connectors.utils import get_connector_info_from_db
11 from app.db.db_session import get_db_session
12
@@ -73,25 +74,30 @@ async def verify_graylog_connection(connector_name: str) -> str:
74 async def send_get_request(
75 endpoint: str,
76 params: Optional[Dict[str, Any]] = None,
76 - connector_name: str = "Graylog",
77 + # connector_name: str = "Graylog",
78 + connector_name: Optional[str] = None,
79 ) -> Dict[str, Any]:
80 """
81 Sends a GET request to the Graylog service.
82
83 Args:
84 endpoint (str): The endpoint to send the GET request to.
83 - params (Optional[Dict[str, Any]], optional): The parameters to send with the GET request. Defaults to None.
84 - connector_name (str, optional): The name of the connector to use. Defaults to "Graylogr".
85 + params (Optional[Dict[str, Any]], optional): The parameters to send with the GET request.
86 + connector_name (Optional[str], optional): The name of the connector to use.
87 + If not provided, uses the current context or defaults to "Graylog".
88
89 Returns:
90 Dict[str, Any]: The response from the GET request.
91 """
89 - logger.info(f"Sending GET request to {endpoint}")
90 - async with get_db_session() as session: # This will correctly enter the context manager
92 + # Use provided connector_name, or fall back to context-based resolution
93 + if connector_name is None:
94 + connector_name = get_current_graylog_connector()
95 +
96 + logger.info(f"Sending GET request to {endpoint} using connector: {connector_name}")
97 + async with get_db_session() as session:
98 attributes = await get_connector_info_from_db(connector_name, session)
99 if attributes is None:
93 - logger.error("No Graylog connector found in the database")
94 - return None
100 + raise HTTPException(status_code=500, detail=f"Connector {connector_name} not found")
101 try:
102 response = requests.get(
103 f"{attributes['connector_url']}{endpoint}",
@@ -125,7 +131,8 @@ async def send_get_request(
131 async def send_post_request(
132 endpoint: str,
133 data: Dict[str, Any] = None,
128 - connector_name: str = "Graylog",
134 + # connector_name: str = "Graylog",
135 + connector_name: Optional[str] = None,
136 ) -> Dict[str, Any]:
137 """
138 Sends a POST request to the Graylog service.
@@ -133,20 +140,21 @@ async def send_post_request(
140 Args:
141 endpoint (str): The endpoint to send the POST request to.
142 data (Dict[str, Any]): The data to send with the POST request.
136 - connector_name (str, optional): The name of the connector to use. Defaults to "Graylog".
143 + connector_name (Optional[str], optional): The name of the connector to use.
144 + If not provided, uses the current context or defaults to "Graylog".
145
146 Returns:
147 Dict[str, Any]: The response from the POST request.
148 """
141 - logger.info(f"Sending POST request to {endpoint}")
142 - async with get_db_session() as session: # This will correctly enter the context manager
149 + # Use provided connector_name, or fall back to context-based resolution
150 + if connector_name is None:
151 + connector_name = get_current_graylog_connector()
152 +
153 + logger.info(f"Sending POST request to {endpoint} using connector: {connector_name}")
154 + async with get_db_session() as session:
155 attributes = await get_connector_info_from_db(connector_name, session)
156 if attributes is None:
145 - logger.error("No Graylog connector found in the database")
146 - return {
147 - "success": False,
148 - "message": "No Graylog connector found in the database",
149 - }
157 + raise HTTPException(status_code=500, detail=f"Connector {connector_name} not found")
158
159 try:
160 response = requests.post(
@@ -205,25 +213,30 @@ async def send_post_request(
213 async def send_delete_request(
214 endpoint: str,
215 params: Optional[Dict[str, Any]] = None,
208 - connector_name: str = "Graylog",
216 + # connector_name: str = "Graylog",
217 + connector_name: Optional[str] = None,
218 ) -> Dict[str, Any]:
219 """
220 Sends a DELETE request to the Graylog service.
221
222 Args:
223 endpoint (str): The endpoint to send the DELETE request to.
215 - params (Optional[Dict[str, Any]], optional): The parameters to send with the DELETE request. Defaults to None.
216 - connector_name (str, optional): The name of the connector to use. Defaults to "Graylog".
224 + params (Optional[Dict[str, Any]], optional): The parameters to send with the DELETE request.
225 + connector_name (Optional[str], optional): The name of the connector to use.
226 + If not provided, uses the current context or defaults to "Graylog".
227
228 Returns:
229 Dict[str, Any]: The response from the DELETE request.
230 """
221 - logger.info(f"Sending DELETE request to {endpoint}")
222 - async with get_db_session() as session: # This will correctly enter the context manager
231 + # Use provided connector_name, or fall back to context-based resolution
232 + if connector_name is None:
233 + connector_name = get_current_graylog_connector()
234 +
235 + logger.info(f"Sending DELETE request to {endpoint} using connector: {connector_name}")
236 + async with get_db_session() as session:
237 attributes = await get_connector_info_from_db(connector_name, session)
238 if attributes is None:
225 - logger.error("No Graylog connector found in the database")
226 - return None
239 + raise HTTPException(status_code=500, detail=f"Connector {connector_name} not found")
240 try:
241 response = requests.delete(
242 f"{attributes['connector_url']}{endpoint}",
@@ -258,20 +271,26 @@ async def send_delete_request(
271 async def send_put_request(
272 endpoint: str,
273 data: Optional[Dict[str, Any]] = None,
261 - connector_name: str = "Graylog",
274 + # connector_name: str = "Graylog",
275 + connector_name: Optional[str] = None,
276 ) -> Dict[str, Any]:
277 """
278 Sends a PUT request to the Graylog service.
279
280 Args:
281 endpoint (str): The endpoint to send the PUT request to.
268 - data (Optional[Dict[str, Any]]): The data to send with the PUT request.
269 - connector_name (str, optional): The name of the connector to use. Defaults to "Graylog".
282 + data (Optional[Dict[str, Any]], optional): The data to send with the PUT request.
283 + connector_name (Optional[str], optional): The name of the connector to use.
284 + If not provided, uses the current context or defaults to "Graylog".
285
286 Returns:
287 Dict[str, Any]: The response from the PUT request.
288 """
274 - logger.info(f"Sending PUT request to {endpoint} with payload {data}")
289 + # Use provided connector_name, or fall back to context-based resolution
290 + if connector_name is None:
291 + connector_name = get_current_graylog_connector()
292 +
293 + logger.info(f"Sending PUT request to {endpoint} using connector: {connector_name}")
294 async with get_db_session() as session: # This will correctly enter the context manager
295 attributes = await get_connector_info_from_db(connector_name, session)
296 if attributes is None:
backend/app/connectors/services.py
+10
@@ -84,6 +84,15 @@ class GraylogService(ConnectorServiceInterface):
84 return await verify_graylog_connection(connector.connector_name)
85
86
87 +# Graylog Network Service (for second Graylog instance)
88 +class GraylogNetworkService(ConnectorServiceInterface):
89 + async def verify_authentication(
90 + self,
91 + connector: ConnectorResponse,
92 + ) -> Optional[ConnectorResponse]:
93 + return await verify_graylog_connection(connector.connector_name)
94 +
95 +
96 # Cortex Service
97 class CortexService(ConnectorServiceInterface):
98 async def verify_authentication(
@@ -205,6 +214,7 @@ def get_connector_service(connector_name: str) -> Type[ConnectorServiceInterface
214 "Wazuh-Indexer": WazuhIndexerService,
215 "Velociraptor": VelociraptorService,
216 "Graylog": GraylogService,
217 + "Graylog-Network": GraylogNetworkService,
218 "Cortex": CortexService,
219 "Shuffle": ShuffleService,
220 "Sublime": SublimeService,
backend/app/db/db_populate.py
+6
@@ -151,6 +151,12 @@ def get_connectors_list():
151 "Connection to VirusTotal. Make sure you have created an API key.",
152 ),
153 ("Portainer", "3", "username_password", "Connection to Portainer.", "PORTAINER_ENDPOINT_ID"),
154 + (
155 + "Graylog-Network",
156 + "5.0.7",
157 + "username_password",
158 + "Connection to Graylog. If you only have one Graylog instance, set this to the same as Graylog.",
159 + ),
160 # ... Add more connectors as needed ...
161 ]
162
backend/app/stack_provisioning/graylog/routes/fortinet.py
+45 -35
@@ -7,6 +7,9 @@ from fastapi import Security
7 from sqlalchemy.ext.asyncio import AsyncSession
8
9 from app.auth.utils import AuthHandler
10 +from app.connectors.graylog.utils.routing import GraylogContext
11 +from app.connectors.graylog.utils.routing import clear_graylog_context
12 +from app.connectors.graylog.utils.routing import set_graylog_context
13 from app.db.db_session import get_db
14 from app.network_connectors.routes import find_customer_network_connector
15 from app.network_connectors.routes import (
@@ -93,43 +96,50 @@ async def provision_fortinet_route(
96 """
97 Provision Fortinet for the customer
98 """
96 - customer_integration_response = await get_customer_integration_response(
97 - provision_fortinet_request.customer_code,
98 - session,
99 - )
99
101 - customer_integration = await find_customer_network_connector(
102 - provision_fortinet_request.customer_code,
103 - provision_fortinet_request.integration_name,
104 - customer_integration_response,
105 - )
100 + # Set the Graylog context for this request - all downstream Graylog calls will use Graylog-Network
101 + set_graylog_context(GraylogContext.NETWORK)
102
107 - fortinet_keys = extract_fortinet_keys(customer_integration)
108 -
109 - if provision_fortinet_request.tcp_enabled and provision_fortinet_request.udp_enabled:
110 - raise HTTPException(
111 - status_code=400,
112 - detail="Both TCP and UDP are enabled. Please choose one of them.",
103 + try:
104 + customer_integration_response = await get_customer_integration_response(
105 + provision_fortinet_request.customer_code,
106 + session,
107 )
114 - elif provision_fortinet_request.tcp_enabled:
115 - protocol_type = "TCP"
116 - elif provision_fortinet_request.udp_enabled:
117 - protocol_type = "UDP"
118 - else:
119 - raise HTTPException(
120 - status_code=400,
121 - detail="Either TCP or UDP should be enabled.",
108 +
109 + customer_integration = await find_customer_network_connector(
110 + provision_fortinet_request.customer_code,
111 + provision_fortinet_request.integration_name,
112 + customer_integration_response,
113 )
114
124 - return await provision_fortinet(
125 - customer_details=FortinetCustomerDetails(
126 - customer_code=provision_fortinet_request.customer_code,
127 - customer_name=customer_integration.customer_name,
128 - protocal_type=protocol_type,
129 - syslog_port=int(fortinet_keys["SYSLOG_PORT"]),
130 - hot_data_retention=provision_fortinet_request.hot_data_retention,
131 - index_replicas=provision_fortinet_request.index_replicas,
132 - ),
133 - keys=ProvisionFortinetKeys(**fortinet_keys),
134 - session=session,
135 - )
115 + fortinet_keys = extract_fortinet_keys(customer_integration)
116 +
117 + if provision_fortinet_request.tcp_enabled and provision_fortinet_request.udp_enabled:
118 + raise HTTPException(
119 + status_code=400,
120 + detail="Both TCP and UDP are enabled. Please choose one of them.",
121 + )
122 + elif provision_fortinet_request.tcp_enabled:
123 + protocol_type = "TCP"
124 + elif provision_fortinet_request.udp_enabled:
125 + protocol_type = "UDP"
126 + else:
127 + raise HTTPException(
128 + status_code=400,
129 + detail="Either TCP or UDP should be enabled.",
130 + )
131 +
132 + return await provision_fortinet(
133 + customer_details=FortinetCustomerDetails(
134 + customer_code=provision_fortinet_request.customer_code,
135 + customer_name=customer_integration.customer_name,
136 + protocal_type=protocol_type,
137 + syslog_port=int(fortinet_keys["SYSLOG_PORT"]),
138 + hot_data_retention=provision_fortinet_request.hot_data_retention,
139 + index_replicas=provision_fortinet_request.index_replicas,
140 + ),
141 + keys=ProvisionFortinetKeys(**fortinet_keys),
142 + session=session,
143 + )
144 + finally:
145 + clear_graylog_context()
backend/app/stack_provisioning/graylog/routes/sentinelone.py
+32 -24
@@ -7,6 +7,9 @@ from fastapi import Security
7 from sqlalchemy.ext.asyncio import AsyncSession
8
9 from app.auth.utils import AuthHandler
10 +from app.connectors.graylog.utils.routing import GraylogContext
11 +from app.connectors.graylog.utils.routing import clear_graylog_context
12 +from app.connectors.graylog.utils.routing import set_graylog_context
13 from app.db.db_session import get_db
14 from app.network_connectors.routes import find_customer_network_connector
15 from app.network_connectors.routes import (
@@ -97,29 +100,34 @@ async def provision_sentinelone_route(
100 """
101 Provision SentinelOne for the customer
102 """
100 - customer_integration_response = await get_customer_integration_response(
101 - provision_sentinelone_request.customer_code,
102 - session,
103 - )
103 + # Set the Graylog context for this request - all downstream Graylog calls will use Graylog-Network
104 + set_graylog_context(GraylogContext.NETWORK)
105 + try:
106 + customer_integration_response = await get_customer_integration_response(
107 + provision_sentinelone_request.customer_code,
108 + session,
109 + )
110
105 - customer_integration = await find_customer_network_connector(
106 - provision_sentinelone_request.customer_code,
107 - provision_sentinelone_request.integration_name,
108 - customer_integration_response,
109 - )
111 + customer_integration = await find_customer_network_connector(
112 + provision_sentinelone_request.customer_code,
113 + provision_sentinelone_request.integration_name,
114 + customer_integration_response,
115 + )
116
111 - sentinelone_keys = extract_sentinelone_keys(customer_integration)
112 -
113 - return await provision_sentinelone(
114 - customer_details=SentinelOneCustomerDetails(
115 - customer_code=provision_sentinelone_request.customer_code,
116 - customer_name=customer_integration.customer_name,
117 - tls_cert_file=sentinelone_keys["TLS_CERT_FILE"],
118 - tls_key_file=sentinelone_keys["TLS_KEY_FILE"],
119 - syslog_port=int(sentinelone_keys["SYSLOG_PORT"]),
120 - hot_data_retention=provision_sentinelone_request.hot_data_retention,
121 - index_replicas=provision_sentinelone_request.index_replicas,
122 - ),
123 - keys=ProvisionSentinelOneKeys(**sentinelone_keys),
124 - session=session,
125 - )
117 + sentinelone_keys = extract_sentinelone_keys(customer_integration)
118 +
119 + return await provision_sentinelone(
120 + customer_details=SentinelOneCustomerDetails(
121 + customer_code=provision_sentinelone_request.customer_code,
122 + customer_name=customer_integration.customer_name,
123 + tls_cert_file=sentinelone_keys["TLS_CERT_FILE"],
124 + tls_key_file=sentinelone_keys["TLS_KEY_FILE"],
125 + syslog_port=int(sentinelone_keys["SYSLOG_PORT"]),
126 + hot_data_retention=provision_sentinelone_request.hot_data_retention,
127 + index_replicas=provision_sentinelone_request.index_replicas,
128 + ),
129 + keys=ProvisionSentinelOneKeys(**sentinelone_keys),
130 + session=session,
131 + )
132 + finally:
133 + clear_graylog_context()
backend/app/stack_provisioning/graylog/routes/sonicwall.py
+37 -26
@@ -7,6 +7,9 @@ from fastapi import Security
7 from sqlalchemy.ext.asyncio import AsyncSession
8
9 from app.auth.utils import AuthHandler
10 +from app.connectors.graylog.utils.routing import GraylogContext
11 +from app.connectors.graylog.utils.routing import clear_graylog_context
12 +from app.connectors.graylog.utils.routing import set_graylog_context
13 from app.db.db_session import get_db
14 from app.network_connectors.routes import find_customer_network_connector
15 from app.network_connectors.routes import (
@@ -83,7 +86,7 @@ def extract_sonicwall_keys(
86 @stack_provisioning_graylog_sonicwall_router.post(
87 "/graylog/provision/sonicwall",
88 response_model=ProvisionSonicwallResponse,
86 - description="Provision SonicWall for the customer.",
89 + description="Provision SonicWall for the customer. Uses Graylog-Network instance for all Graylog operations.",
90 dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
91 )
92 async def provision_sonicwall_route(
@@ -91,31 +94,39 @@ async def provision_sonicwall_route(
94 session: AsyncSession = Depends(get_db),
95 ) -> ProvisionSonicwallResponse:
96 """
94 - Provision SonicWall for the customer
97 + Provision SonicWall for the customer.
98 + Uses Graylog-Network instance for all Graylog operations.
99 """
96 - customer_integration_response = await get_customer_integration_response(
97 - provision_sonicwall_request.customer_code,
98 - session,
99 - )
100 + # Set the Graylog context for this request - all downstream Graylog calls will use Graylog-Network
101 + set_graylog_context(GraylogContext.NETWORK)
102
101 - customer_integration = await find_customer_network_connector(
102 - provision_sonicwall_request.customer_code,
103 - provision_sonicwall_request.integration_name,
104 - customer_integration_response,
105 - )
103 + try:
104 + customer_integration_response = await get_customer_integration_response(
105 + provision_sonicwall_request.customer_code,
106 + session,
107 + )
108
107 - sonicwall_keys = extract_sonicwall_keys(customer_integration)
108 -
109 - return await provision_sonicwall(
110 - customer_details=SonicwallCustomerDetails(
111 - customer_code=provision_sonicwall_request.customer_code,
112 - customer_name=customer_integration.customer_name,
113 - tls_cert_file=sonicwall_keys["TLS_CERT_FILE"],
114 - tls_key_file=sonicwall_keys["TLS_KEY_FILE"],
115 - syslog_port=int(sonicwall_keys["SYSLOG_PORT"]),
116 - hot_data_retention=provision_sonicwall_request.hot_data_retention,
117 - index_replicas=provision_sonicwall_request.index_replicas,
118 - ),
119 - keys=ProvisionSonicwallKeys(**sonicwall_keys),
120 - session=session,
121 - )
109 + customer_integration = await find_customer_network_connector(
110 + provision_sonicwall_request.customer_code,
111 + provision_sonicwall_request.integration_name,
112 + customer_integration_response,
113 + )
114 +
115 + sonicwall_keys = extract_sonicwall_keys(customer_integration)
116 +
117 + return await provision_sonicwall(
118 + customer_details=SonicwallCustomerDetails(
119 + customer_code=provision_sonicwall_request.customer_code,
120 + customer_name=customer_integration.customer_name,
121 + tls_cert_file=sonicwall_keys["TLS_CERT_FILE"],
122 + tls_key_file=sonicwall_keys["TLS_KEY_FILE"],
123 + syslog_port=int(sonicwall_keys["SYSLOG_PORT"]),
124 + hot_data_retention=provision_sonicwall_request.hot_data_retention,
125 + index_replicas=provision_sonicwall_request.index_replicas,
126 + ),
127 + keys=ProvisionSonicwallKeys(**sonicwall_keys),
128 + session=session,
129 + )
130 + finally:
131 + # Always clear the context when done
132 + clear_graylog_context()