@cryptotaxi247 / CoPilot / commits / c0f53deb

Shuffle singul testing (#461)

* Fix grafana_dashboard_folder_id assignment and add customer network connectors metadata handling * Add deletion of Grafana datasource in delete_integration function * precommit-fixes * Add conditional check for Grafana datasource deletion in delete_integration function * Add metadata retrieval and update endpoints for integrations and network connectors * precommit fixes * Remove commented-out delete_integration_meta endpoint from routes.py * precommit fixes * Add defender for endpoint provisioning functionality

taylor_socfortress committed Jun 20, 2025 at 15:40 UTC c0f53debf5ffc5efa7a4723bdffaddd589316b05
5 files changed +369 -41
backend/app/integrations/defender_for_endpoint/services/provision.py
+1 -1
@@ -367,7 +367,7 @@ async def provision_defender_for_endpoint(
367 datasourceUid=customer_network_connector_meta.grafana_datasource_uid,
368 ),
369 )
370 - customer_network_connector_meta.grafana_dashboard_folder_id = grafana_folder.uid
370 + customer_network_connector_meta.grafana_dashboard_folder_id = grafana_folder.id
371 await insert_into_customer_network_connectors_meta_table(
372 customer_network_connectors_meta=customer_network_connector_meta,
373 session=session,
backend/app/integrations/routes.py
+336 -38
@@ -17,6 +17,7 @@ from app.auth.utils import AuthHandler
17 from app.connectors.grafana.services.folders import delete_folder
18 from app.connectors.graylog.services.management import delete_index_by_id
19 from app.connectors.graylog.services.streams import delete_stream
20 +from app.customer_provisioning.services.grafana import delete_grafana_datasource
21 from app.db.db_session import get_db
22 from app.db.universal_models import Customers
23 from app.db.universal_models import CustomersMeta
@@ -48,9 +49,21 @@ from app.integrations.schema import CustomerIntegrationsResponse
49 from app.integrations.schema import DeleteCustomerIntegration
50 from app.integrations.schema import IntegrationWithAuthKeys
51 from app.integrations.schema import UpdateCustomerIntegration
52 +from app.integrations.schema import UpdateMetaAutoRequest
53 +from app.integrations.schema import UpdateMetaResponse
54 +from app.network_connectors.models.network_connectors import (
55 + CustomerNetworkConnectorsMeta,
56 +)
57
58 integration_settings_router = APIRouter()
59
60 +NETWORK_INTEGRATIONS = [
61 + "DefenderForEndpoint",
62 + "BITDEFENDER",
63 + "CROWDSTRIKE",
64 + # Add other network integrations as needed
65 +]
66 +
67
68 async def fetch_available_integrations(session: AsyncSession):
69 """
@@ -937,6 +950,95 @@ async def delete_customer_integration_meta(session: AsyncSession, customer_code:
950 )
951
952
953 +async def fetch_customer_network_connectors_meta(session: AsyncSession, customer_code: str, network_connector_name: str):
954 + """
955 + Fetches customer network connectors metadata from the database.
956 + """
957 + stmt = select(CustomerNetworkConnectorsMeta).where(
958 + CustomerNetworkConnectorsMeta.customer_code == customer_code,
959 + CustomerNetworkConnectorsMeta.network_connector_name == network_connector_name,
960 + )
961 + result = await session.execute(stmt)
962 + return result.scalars().first()
963 +
964 +
965 +async def delete_customer_network_connectors_meta(session: AsyncSession, customer_code: str, network_connector_name: str):
966 + """
967 + Deletes customer network connectors metadata from the database.
968 + """
969 + await session.execute(
970 + delete(CustomerNetworkConnectorsMeta).where(
971 + CustomerNetworkConnectorsMeta.customer_code == customer_code,
972 + CustomerNetworkConnectorsMeta.network_connector_name == network_connector_name,
973 + ),
974 + )
975 +
976 +
977 +# @integration_settings_router.delete(
978 +# "/delete_integration",
979 +# response_model=CustomerIntegrationDeleteResponse,
980 +# description="Delete a customer integration.",
981 +# dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
982 +# )
983 +# async def delete_integration(
984 +# delete_customer_integration: DeleteCustomerIntegration,
985 +# session: AsyncSession = Depends(get_db),
986 +# ):
987 +# customer_code = delete_customer_integration.customer_code
988 +# integration_name = delete_customer_integration.integration_name
989 +
990 +# results = await get_customer_and_service_ids(
991 +# session,
992 +# customer_code,
993 +# integration_name,
994 +# )
995 +# # Check if results is not empty
996 +# if results:
997 +# # Unpack the first tuple in results
998 +# customer_id, integration_service_id = results[0]
999 +# else:
1000 +# # Handle the case where results is empty
1001 +# raise HTTPException(status_code=404, detail="Customer integration not found")
1002 +
1003 +# subscription_ids = await get_subscription_ids(
1004 +# session,
1005 +# customer_id,
1006 +# integration_service_id,
1007 +# )
1008 +# if not subscription_ids:
1009 +# raise HTTPException(
1010 +# status_code=404,
1011 +# detail="No subscriptions found for customer integration",
1012 +# )
1013 +
1014 +# stream_id = (await fetch_customer_integration_meta(session, customer_code, integration_name)).graylog_stream_id
1015 +# logger.info(f"stream_id: {stream_id}")
1016 +# await delete_stream(stream_id=stream_id)
1017 +
1018 +# index_id = (await fetch_customer_integration_meta(session, customer_code, integration_name)).graylog_index_id
1019 +# logger.info(f"index_id: {index_id}")
1020 +# await delete_index_by_id(index_id=index_id)
1021 +
1022 +# # Delete the folder in Grafana
1023 +# grafana_org_id = (await fetch_customer_integration_meta(session, customer_code, integration_name)).grafana_org_id
1024 +# grafana_dashboard_folder_id = (
1025 +# await fetch_customer_integration_meta(session, customer_code, integration_name)
1026 +# ).grafana_dashboard_folder_id
1027 +
1028 +# await delete_folder(grafana_org_id, int(grafana_dashboard_folder_id))
1029 +
1030 +# await delete_metadata(session, subscription_ids)
1031 +# await delete_subscriptions(session, subscription_ids)
1032 +# await delete_configs(session, integration_service_id)
1033 +# await delete_integration_service(session, integration_service_id)
1034 +# await delete_customer_integration_record(session, customer_id)
1035 +# await delete_customer_integration_meta(session, customer_code, integration_name)
1036 +
1037 +# await session.commit()
1038 +
1039 +# return generate_decommission_response(customer_code, integration_name)
1040 +
1041 +
1042 @integration_settings_router.delete(
1043 "/delete_integration",
1044 response_model=CustomerIntegrationDeleteResponse,
@@ -950,6 +1052,9 @@ async def delete_integration(
1052 customer_code = delete_customer_integration.customer_code
1053 integration_name = delete_customer_integration.integration_name
1054
1055 + # Check if this is a network integration
1056 + is_network_integration = integration_name in NETWORK_INTEGRATIONS
1057 +
1058 results = await get_customer_and_service_ids(
1059 session,
1060 customer_code,
@@ -974,28 +1079,48 @@ async def delete_integration(
1079 detail="No subscriptions found for customer integration",
1080 )
1081
977 - stream_id = (await fetch_customer_integration_meta(session, customer_code, integration_name)).graylog_stream_id
1082 + # Fetch metadata from appropriate table based on integration type
1083 + if is_network_integration:
1084 + meta_data = await fetch_customer_network_connectors_meta(session, customer_code, integration_name)
1085 + else:
1086 + meta_data = await fetch_customer_integration_meta(session, customer_code, integration_name)
1087 +
1088 + if not meta_data:
1089 + raise HTTPException(status_code=404, detail=f"Metadata not found for {integration_name} integration")
1090 +
1091 + # Delete stream and index using metadata
1092 + stream_id = meta_data.graylog_stream_id
1093 logger.info(f"stream_id: {stream_id}")
1094 await delete_stream(stream_id=stream_id)
1095
981 - index_id = (await fetch_customer_integration_meta(session, customer_code, integration_name)).graylog_index_id
1096 + index_id = meta_data.graylog_index_id
1097 logger.info(f"index_id: {index_id}")
1098 await delete_index_by_id(index_id=index_id)
1099
1100 # Delete the folder in Grafana
986 - grafana_org_id = (await fetch_customer_integration_meta(session, customer_code, integration_name)).grafana_org_id
987 - grafana_dashboard_folder_id = (
988 - await fetch_customer_integration_meta(session, customer_code, integration_name)
989 - ).grafana_dashboard_folder_id
1101 + grafana_org_id = meta_data.grafana_org_id
1102 + grafana_dashboard_folder_id = meta_data.grafana_dashboard_folder_id
1103
1104 await delete_folder(grafana_org_id, int(grafana_dashboard_folder_id))
1105
1106 + # Delete the grafana datasource
1107 + if is_network_integration:
1108 + await delete_grafana_datasource(
1109 + organization_id=grafana_org_id,
1110 + datasource_uid=meta_data.grafana_datasource_uid,
1111 + )
1112 +
1113 await delete_metadata(session, subscription_ids)
1114 await delete_subscriptions(session, subscription_ids)
1115 await delete_configs(session, integration_service_id)
1116 await delete_integration_service(session, integration_service_id)
1117 await delete_customer_integration_record(session, customer_id)
998 - await delete_customer_integration_meta(session, customer_code, integration_name)
1118 +
1119 + # Delete metadata from appropriate table
1120 + if is_network_integration:
1121 + await delete_customer_network_connectors_meta(session, customer_code, integration_name)
1122 + else:
1123 + await delete_customer_integration_meta(session, customer_code, integration_name)
1124
1125 await session.commit()
1126
@@ -1067,34 +1192,207 @@ async def get_customer_by_auth_key(
1192 raise HTTPException(status_code=500, detail=f"Failed to look up customer: {str(e)}")
1193
1194
1070 -# @integration_settings_router.delete(
1071 -# "/delete_integration_meta",
1072 -# response_model=CustomerIntegrationsMetaResponse,
1073 -# description="Delete a customer integration metadata.",
1074 -# dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
1075 -# )
1076 -# async def delete_integration_meta(
1077 -# customer_integration_meta: CustomerIntegrationsMetaSchema,
1078 -# session: AsyncSession = Depends(get_db),
1079 -# ):
1080 -# """
1081 -# Endpoint to delete a customer integration metadata.
1082 -# """
1083 -# try:
1084 -# stmt = delete(CustomerIntegrationsMeta).where(
1085 -# CustomerIntegrationsMeta.customer_code == customer_integration_meta.customer_code,
1086 -# CustomerIntegrationsMeta.integration_name == customer_integration_meta.integration_name,
1087 -# )
1088 -# await session.execute(stmt)
1089 -# await session.commit()
1090 -# return CustomerIntegrationsMetaResponse(
1091 -# message="Customer integration metadata successfully deleted.",
1092 -# success=True,
1093 -# )
1094 -# except Exception as e:
1095 -# logger.error(f"Error while deleting customer integration metadata: {e}")
1096 -# return CustomerIntegrationsMetaResponse(
1097 -# customer_integrations_meta=None,
1098 -# message="Error while deleting customer integration metadata.",
1099 -# success=False,
1100 -# )
1195 +@integration_settings_router.get(
1196 + "/meta_auto/{customer_code}/{integration_name}",
1197 + description="Get integration or network connector metadata automatically based on integration name",
1198 + dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
1199 +)
1200 +async def get_meta_auto(
1201 + customer_code: str,
1202 + integration_name: str,
1203 + session: AsyncSession = Depends(get_db),
1204 +):
1205 + """
1206 + Automatically retrieve metadata from the appropriate table based on integration name.
1207 +
1208 + This route checks if the integration is in the NETWORK_INTEGRATIONS list and
1209 + fetches from the appropriate table accordingly.
1210 +
1211 + Args:
1212 + customer_code (str): The customer code to filter by
1213 + integration_name (str): The integration/connector name to filter by
1214 + session (AsyncSession): Database session
1215 +
1216 + Returns:
1217 + dict: The metadata record from the appropriate table
1218 + """
1219 + logger.info(f"Fetching metadata for customer {customer_code} and integration {integration_name}")
1220 +
1221 + is_network_integration = integration_name in NETWORK_INTEGRATIONS
1222 +
1223 + try:
1224 + if is_network_integration:
1225 + # Fetch from network connectors table
1226 + stmt = select(CustomerNetworkConnectorsMeta).where(
1227 + CustomerNetworkConnectorsMeta.customer_code == customer_code,
1228 + CustomerNetworkConnectorsMeta.network_connector_name == integration_name,
1229 + )
1230 + result = await session.execute(stmt)
1231 + meta_record = result.scalars().first()
1232 +
1233 + if not meta_record:
1234 + raise HTTPException(
1235 + status_code=404,
1236 + detail=f"Network connector metadata not found for customer {customer_code} and connector {integration_name}",
1237 + )
1238 +
1239 + logger.info(f"Successfully retrieved network connector metadata for {customer_code}/{integration_name}")
1240 +
1241 + else:
1242 + # Fetch from regular integrations table
1243 + stmt = select(CustomerIntegrationsMeta).where(
1244 + CustomerIntegrationsMeta.customer_code == customer_code,
1245 + CustomerIntegrationsMeta.integration_name == integration_name,
1246 + )
1247 + result = await session.execute(stmt)
1248 + meta_record = result.scalars().first()
1249 +
1250 + if not meta_record:
1251 + raise HTTPException(
1252 + status_code=404,
1253 + detail=f"Integration metadata not found for customer {customer_code} and integration {integration_name}",
1254 + )
1255 +
1256 + logger.info(f"Successfully retrieved integration metadata for {customer_code}/{integration_name}")
1257 +
1258 + # Convert SQLModel to dict for response
1259 + return {
1260 + "success": True,
1261 + "message": f"Successfully retrieved metadata for {customer_code}/{integration_name}",
1262 + "data": meta_record.dict() if hasattr(meta_record, "dict") else meta_record.__dict__,
1263 + "table_type": "network_connector" if is_network_integration else "integration",
1264 + }
1265 +
1266 + except HTTPException:
1267 + raise
1268 + except Exception as e:
1269 + logger.error(f"Error retrieving metadata: {str(e)}")
1270 + raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
1271 +
1272 +
1273 +@integration_settings_router.put(
1274 + "/update_meta_auto",
1275 + response_model=UpdateMetaResponse,
1276 + description="Automatically update integration or network connector metadata based on integration name",
1277 + dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
1278 +)
1279 +async def update_meta_auto(
1280 + update_request: UpdateMetaAutoRequest,
1281 + session: AsyncSession = Depends(get_db),
1282 +):
1283 + """
1284 + Automatically determine which table to update based on integration name.
1285 +
1286 + This route checks if the integration is in the NETWORK_INTEGRATIONS list and
1287 + updates the appropriate table accordingly.
1288 + """
1289 + is_network_integration = update_request.integration_name in NETWORK_INTEGRATIONS
1290 +
1291 + try:
1292 + if is_network_integration:
1293 + # Check if record exists in network connectors table
1294 + stmt = select(CustomerNetworkConnectorsMeta).where(
1295 + CustomerNetworkConnectorsMeta.customer_code == update_request.customer_code,
1296 + CustomerNetworkConnectorsMeta.network_connector_name == update_request.integration_name,
1297 + )
1298 + result = await session.execute(stmt)
1299 + existing_record = result.scalars().first()
1300 +
1301 + if not existing_record:
1302 + raise HTTPException(
1303 + status_code=404,
1304 + detail=f"Network connector metadata not found for customer {update_request.customer_code} and connector {update_request.integration_name}",
1305 + )
1306 +
1307 + # Update network connector metadata
1308 + update_data = {}
1309 + if update_request.graylog_input_id is not None:
1310 + update_data["graylog_input_id"] = update_request.graylog_input_id
1311 + if update_request.graylog_index_id is not None:
1312 + update_data["graylog_index_id"] = update_request.graylog_index_id
1313 + if update_request.graylog_stream_id is not None:
1314 + update_data["graylog_stream_id"] = update_request.graylog_stream_id
1315 + if update_request.graylog_pipeline_id is not None:
1316 + update_data["graylog_pipeline_id"] = update_request.graylog_pipeline_id
1317 + if update_request.graylog_content_pack_input_id is not None:
1318 + update_data["graylog_content_pack_input_id"] = update_request.graylog_content_pack_input_id
1319 + if update_request.graylog_content_pack_stream_id is not None:
1320 + update_data["graylog_content_pack_stream_id"] = update_request.graylog_content_pack_stream_id
1321 + if update_request.grafana_org_id is not None:
1322 + update_data["grafana_org_id"] = update_request.grafana_org_id
1323 + if update_request.grafana_dashboard_folder_id is not None:
1324 + update_data["grafana_dashboard_folder_id"] = update_request.grafana_dashboard_folder_id
1325 + if update_request.grafana_datasource_uid is not None:
1326 + update_data["grafana_datasource_uid"] = update_request.grafana_datasource_uid
1327 +
1328 + if update_data:
1329 + update_stmt = (
1330 + update(CustomerNetworkConnectorsMeta)
1331 + .where(
1332 + CustomerNetworkConnectorsMeta.customer_code == update_request.customer_code,
1333 + CustomerNetworkConnectorsMeta.network_connector_name == update_request.integration_name,
1334 + )
1335 + .values(**update_data)
1336 + )
1337 + await session.execute(update_stmt)
1338 +
1339 + else:
1340 + # Check if record exists in integrations table
1341 + stmt = select(CustomerIntegrationsMeta).where(
1342 + CustomerIntegrationsMeta.customer_code == update_request.customer_code,
1343 + CustomerIntegrationsMeta.integration_name == update_request.integration_name,
1344 + )
1345 + result = await session.execute(stmt)
1346 + existing_record = result.scalars().first()
1347 +
1348 + if not existing_record:
1349 + raise HTTPException(
1350 + status_code=404,
1351 + detail=f"Integration metadata not found for customer {update_request.customer_code} and integration {update_request.integration_name}",
1352 + )
1353 +
1354 + # Update regular integration metadata
1355 + update_data = {}
1356 + if update_request.graylog_input_id is not None:
1357 + update_data["graylog_input_id"] = update_request.graylog_input_id
1358 + if update_request.graylog_index_id is not None:
1359 + update_data["graylog_index_id"] = update_request.graylog_index_id
1360 + if update_request.graylog_stream_id is not None:
1361 + update_data["graylog_stream_id"] = update_request.graylog_stream_id
1362 + if update_request.grafana_org_id is not None:
1363 + update_data["grafana_org_id"] = update_request.grafana_org_id
1364 + if update_request.grafana_dashboard_folder_id is not None:
1365 + update_data["grafana_dashboard_folder_id"] = update_request.grafana_dashboard_folder_id
1366 +
1367 + if update_data:
1368 + update_stmt = (
1369 + update(CustomerIntegrationsMeta)
1370 + .where(
1371 + CustomerIntegrationsMeta.customer_code == update_request.customer_code,
1372 + CustomerIntegrationsMeta.integration_name == update_request.integration_name,
1373 + )
1374 + .values(**update_data)
1375 + )
1376 + await session.execute(update_stmt)
1377 +
1378 + if not update_data:
1379 + return UpdateMetaResponse(success=False, message="No fields provided for update")
1380 +
1381 + await session.commit()
1382 +
1383 + table_type = "network connector" if is_network_integration else "integration"
1384 + logger.info(
1385 + f"Updated {table_type} metadata for customer {update_request.customer_code}, {table_type} {update_request.integration_name}",
1386 + )
1387 +
1388 + return UpdateMetaResponse(
1389 + success=True,
1390 + message=f"Successfully updated {table_type} metadata for {update_request.customer_code}/{update_request.integration_name}",
1391 + )
1392 +
1393 + except HTTPException:
1394 + raise
1395 + except Exception as e:
1396 + await session.rollback()
1397 + logger.error(f"Error updating metadata: {str(e)}")
1398 + raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
backend/app/integrations/schema.py
+19
@@ -251,3 +251,22 @@ class CustomerByAuthKeyResponse(BaseModel):
251 auth_key_name: str
252 success: bool = True
253 message: str = "Customer found successfully."
254 +
255 +
256 +class UpdateMetaAutoRequest(BaseModel):
257 + customer_code: str = Field(..., description="Customer code identifier")
258 + integration_name: str = Field(..., description="Integration or network connector name")
259 + graylog_input_id: Optional[str] = Field(None, description="Graylog input ID")
260 + graylog_index_id: Optional[str] = Field(None, description="Graylog index ID")
261 + graylog_stream_id: Optional[str] = Field(None, description="Graylog stream ID")
262 + graylog_pipeline_id: Optional[str] = Field(None, description="Graylog pipeline ID (network connectors only)")
263 + graylog_content_pack_input_id: Optional[str] = Field(None, description="Graylog content pack input ID (network connectors only)")
264 + graylog_content_pack_stream_id: Optional[str] = Field(None, description="Graylog content pack stream ID (network connectors only)")
265 + grafana_org_id: Optional[str] = Field(None, description="Grafana organization ID")
266 + grafana_dashboard_folder_id: Optional[str] = Field(None, description="Grafana dashboard folder ID")
267 + grafana_datasource_uid: Optional[str] = Field(None, description="Grafana datasource UID (network connectors only)")
268 +
269 +
270 +class UpdateMetaResponse(BaseModel):
271 + success: bool
272 + message: str
frontend/src/api/endpoints/integrations.ts
+7 -1
@@ -109,6 +109,12 @@ export default {
109 integration_name: integrationName || "Cato",
110 time_interval: 15
111 })
112 - }
112 + },
113 + defenderForEndpointProvision(customerCode: string, integrationName: string) {
114 + return HttpClient.post<FlaskBaseResponse>(`/defender_for_endpoint/provision`, {
115 + customer_code: customerCode,
116 + integration_name: integrationName || "DefenderForEndpoint"
117 + })
118 + }
119 // #endregion
120 }
frontend/src/components/customers/integrations/CustomerIntegrationActions.vue
+6 -1
@@ -57,6 +57,7 @@ const isDuo = computed(() => serviceName.value === "DUO")
57 const isDarktrace = computed(() => serviceName.value === "Darktrace")
58 const isBitdefender = computed(() => serviceName.value === "BitDefender")
59 const isCato = computed(() => serviceName.value === "CATO")
60 +const isDefenderForEndpoint = computed(() => serviceName.value === "DefenderForEndpoint")
61 const isDeployEnabled = computed(
62 () =>
63 (isOffice365.value ||
@@ -65,7 +66,8 @@ const isDeployEnabled = computed(
66 isDuo.value ||
67 isDarktrace.value ||
68 isBitdefender.value ||
68 - isCato.value) &&
69 + isCato.value ||
70 + isDefenderForEndpoint.value) &&
71 !integration.deployed
72 )
73
@@ -101,6 +103,9 @@ function provision() {
103 if (isCato.value) {
104 apiCall = Api.integrations.catoProvision(customerCode.value, serviceName.value)
105 }
106 + if (isDefenderForEndpoint.value) {
107 + apiCall = Api.integrations.defenderForEndpointProvision(customerCode.value, serviceName.value)
108 + }
109
110 if (!apiCall) {
111 return