@cryptotaxi247 / CoPilot / commits / 0e98469f

Sca overview backend (#507)

* Add SCA routes and implement overview and statistics endpoints * precommit-fixes

taylor_socfortress committed Sep 11, 2025 at 14:38 UTC 0e98469f124fc2eafea4488eba2d795fd1cc4513
8 files changed +599
backend/app/agents/sca/__init__.py new
+1
@@ -0,0 +1 @@
1 +# SCA Package - Security Configuration Assessment Overview
backend/app/agents/sca/routes/__init__.py new
+1
@@ -0,0 +1 @@
1 +# SCA Routes Package
backend/app/agents/sca/routes/sca.py new
+173
@@ -0,0 +1,173 @@
1 +from typing import Optional
2 +
3 +from fastapi import APIRouter
4 +from fastapi import Depends
5 +from fastapi import HTTPException
6 +from fastapi import Query
7 +from fastapi import Security
8 +from loguru import logger
9 +from sqlalchemy.ext.asyncio import AsyncSession
10 +
11 +from app.agents.sca.schema.sca import ScaOverviewResponse
12 +from app.agents.sca.schema.sca import ScaStatsResponse
13 +from app.agents.sca.services.sca import get_sca_statistics
14 +from app.agents.sca.services.sca import search_sca_overview
15 +from app.auth.routes.auth import AuthHandler
16 +from app.db.db_session import get_db
17 +
18 +# Create router for SCA overview endpoints
19 +sca_router = APIRouter()
20 +
21 +
22 +@sca_router.get(
23 + "/overview",
24 + response_model=ScaOverviewResponse,
25 + description="Search SCA results across all agents with filtering and pagination",
26 + dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
27 +)
28 +async def search_sca_results_overview(
29 + customer_code: Optional[str] = Query(None, description="Filter by customer code"),
30 + agent_name: Optional[str] = Query(None, description="Filter by agent hostname"),
31 + policy_id: Optional[str] = Query(None, description="Filter by specific policy ID"),
32 + policy_name: Optional[str] = Query(None, description="Filter by policy name (partial matching)"),
33 + min_score: Optional[int] = Query(None, description="Filter by minimum score (0-100)", ge=0, le=100),
34 + max_score: Optional[int] = Query(None, description="Filter by maximum score (0-100)", ge=0, le=100),
35 + page: int = Query(1, description="Page number for pagination", ge=1),
36 + page_size: int = Query(50, description="Number of results per page", ge=1, le=1000),
37 + db: AsyncSession = Depends(get_db),
38 +) -> ScaOverviewResponse:
39 + """
40 + Search SCA (Security Configuration Assessment) results across all agents.
41 +
42 + This endpoint provides a comprehensive overview of SCA compliance across your
43 + infrastructure by querying all agents and their SCA policy results.
44 +
45 + **Features:**
46 + - Real-time data collection from Wazuh Manager for all agents
47 + - Advanced filtering by customer, agent, policy, and compliance scores
48 + - Efficient pagination for large result sets
49 + - Comprehensive statistics and aggregations
50 + - No database storage required - direct from Wazuh Manager
51 +
52 + **Use Cases:**
53 + - Get organization-wide SCA compliance overview
54 + - Identify agents with poor compliance scores
55 + - Monitor specific security policies across all systems
56 + - Track compliance trends and improvements
57 +
58 + **Performance:**
59 + - Efficiently queries multiple agents in parallel where possible
60 + - Automatic error handling for unavailable agents
61 + - Optimized data collection and processing
62 + - Smart filtering to reduce data transfer
63 +
64 + **Filtering Options:**
65 + - **customer_code**: Filter by specific customer/organization
66 + - **agent_name**: Filter by specific agent hostname
67 + - **policy_id**: Search for specific policy ID (exact match)
68 + - **policy_name**: Filter by policy name (supports partial matching)
69 + - **min_score**: Filter by minimum compliance score (0-100)
70 + - **max_score**: Filter by maximum compliance score (0-100)
71 +
72 + **Response Statistics:**
73 + - **total_agents**: Number of unique agents with SCA data
74 + - **total_policies**: Number of unique policies across all agents
75 + - **average_score**: Average compliance score across all results
76 + - **total_checks/passes/fails/invalid**: Aggregated counts across all agents
77 +
78 + **Pagination:**
79 + - **page**: Page number (starts at 1)
80 + - **page_size**: Results per page (1-1000, default: 50)
81 +
82 + Args:
83 + customer_code: Optional customer code filter
84 + agent_name: Optional agent hostname filter
85 + policy_id: Optional policy ID filter (exact match)
86 + policy_name: Optional policy name filter (partial matching)
87 + min_score: Optional minimum compliance score filter
88 + max_score: Optional maximum compliance score filter
89 + page: Page number for pagination
90 + page_size: Number of results per page
91 + db: Database session
92 +
93 + Returns:
94 + ScaOverviewResponse: Paginated SCA results with comprehensive statistics
95 + """
96 + logger.info(
97 + f"Searching SCA overview with filters: "
98 + f"customer_code={customer_code}, agent_name={agent_name}, "
99 + f"policy_id={policy_id}, policy_name={policy_name}, "
100 + f"min_score={min_score}, max_score={max_score}, "
101 + f"page={page}, page_size={page_size}",
102 + )
103 +
104 + try:
105 + result = await search_sca_overview(
106 + db_session=db,
107 + customer_code=customer_code,
108 + agent_name=agent_name,
109 + policy_id=policy_id,
110 + policy_name=policy_name,
111 + min_score=min_score,
112 + max_score=max_score,
113 + page=page,
114 + page_size=page_size,
115 + )
116 + return result
117 +
118 + except Exception as e:
119 + logger.error(f"Error in SCA overview search endpoint: {e}")
120 + raise HTTPException(status_code=500, detail=f"Failed to search SCA results: {e}")
121 +
122 +
123 +@sca_router.get(
124 + "/stats",
125 + response_model=ScaStatsResponse,
126 + description="Get SCA statistics across all agents or for a specific customer",
127 + dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
128 +)
129 +async def get_sca_stats(
130 + customer_code: Optional[str] = Query(None, description="Filter by customer code"),
131 + db: AsyncSession = Depends(get_db),
132 +) -> ScaStatsResponse:
133 + """
134 + Get comprehensive SCA (Security Configuration Assessment) statistics.
135 +
136 + This endpoint provides high-level statistics about SCA compliance across
137 + your infrastructure, helping you understand overall security posture.
138 +
139 + **Features:**
140 + - Organization-wide or customer-specific statistics
141 + - Real-time data collection from all agents
142 + - Aggregated compliance metrics
143 + - Breakdown by customer when viewing all data
144 +
145 + **Statistics Provided:**
146 + - **total_agents_with_sca**: Number of agents that have SCA data
147 + - **total_policies**: Number of unique security policies across all agents
148 + - **average_score_across_all**: Overall average compliance score
149 + - **total_checks/passes/fails/invalid**: Sum of all checks across all agents
150 + - **by_customer**: Detailed breakdown when viewing all customers
151 +
152 + **Use Cases:**
153 + - Executive dashboards and reporting
154 + - Compliance trend monitoring
155 + - Cross-customer comparison (for MSPs)
156 + - Infrastructure security health checks
157 +
158 + Args:
159 + customer_code: Optional customer code to filter statistics by
160 + db: Database session
161 +
162 + Returns:
163 + ScaStatsResponse: Comprehensive SCA statistics
164 + """
165 + logger.info(f"Getting SCA statistics for customer: {customer_code or 'all customers'}")
166 +
167 + try:
168 + result = await get_sca_statistics(db_session=db, customer_code=customer_code)
169 + return result
170 +
171 + except Exception as e:
172 + logger.error(f"Error getting SCA statistics: {e}")
173 + raise HTTPException(status_code=500, detail=f"Failed to get SCA statistics: {e}")
backend/app/agents/sca/schema/__init__.py new
+1
@@ -0,0 +1 @@
1 +# SCA Schema Package
backend/app/agents/sca/schema/sca.py new
+80
@@ -0,0 +1,80 @@
1 +from typing import List
2 +from typing import Optional
3 +
4 +from pydantic import BaseModel
5 +from pydantic import Field
6 +
7 +
8 +class AgentScaOverviewItem(BaseModel):
9 + """Individual SCA item from overview search results"""
10 +
11 + agent_id: str
12 + agent_name: str
13 + customer_code: Optional[str] = None
14 + policy_id: str
15 + policy_name: str
16 + description: str
17 + total_checks: int
18 + pass_count: int = Field(..., alias="pass")
19 + fail_count: int = Field(..., alias="fail")
20 + invalid_count: int = Field(..., alias="invalid")
21 + score: int
22 + start_scan: str
23 + end_scan: str
24 + references: Optional[str] = None
25 + hash_file: Optional[str] = None
26 +
27 + class Config:
28 + allow_population_by_field_name = True
29 +
30 +
31 +class ScaOverviewResponse(BaseModel):
32 + """Response schema for SCA overview search results with pagination and stats"""
33 +
34 + sca_results: List[AgentScaOverviewItem]
35 + total_count: int
36 + # Summary stats across all results
37 + total_agents: int
38 + total_policies: int
39 + average_score: float
40 + total_checks_all_agents: int
41 + total_passes_all_agents: int
42 + total_fails_all_agents: int
43 + total_invalid_all_agents: int
44 + # Pagination
45 + page: int
46 + page_size: int
47 + total_pages: int
48 + has_next: bool
49 + has_previous: bool
50 + success: bool
51 + message: str
52 + filters_applied: dict = {}
53 +
54 +
55 +class ScaOverviewRequest(BaseModel):
56 + """Request schema for searching SCA results across agents"""
57 +
58 + customer_code: Optional[str] = Field(None, description="Filter by customer code")
59 + agent_name: Optional[str] = Field(None, description="Filter by agent hostname")
60 + policy_id: Optional[str] = Field(None, description="Filter by specific policy ID")
61 + policy_name: Optional[str] = Field(None, description="Filter by policy name (partial matching)")
62 + min_score: Optional[int] = Field(None, description="Filter by minimum score", ge=0, le=100)
63 + max_score: Optional[int] = Field(None, description="Filter by maximum score", ge=0, le=100)
64 + page: int = Field(1, description="Page number for pagination", ge=1)
65 + page_size: int = Field(50, description="Number of results per page", ge=1, le=1000)
66 +
67 +
68 +class ScaStatsResponse(BaseModel):
69 + """Response schema for SCA statistics"""
70 +
71 + total_agents_with_sca: int
72 + total_policies: int
73 + average_score_across_all: float
74 + total_checks_all_agents: int
75 + total_passes_all_agents: int
76 + total_fails_all_agents: int
77 + total_invalid_all_agents: int
78 + by_customer: dict = {}
79 + success: bool
80 + message: str
backend/app/agents/sca/services/__init__.py new
+1
@@ -0,0 +1 @@
1 +# SCA Services Package
backend/app/agents/sca/services/sca.py new
+340
@@ -0,0 +1,340 @@
1 +from typing import List
2 +from typing import Optional
3 +
4 +from fastapi import HTTPException
5 +from loguru import logger
6 +from sqlalchemy import select
7 +from sqlalchemy.ext.asyncio import AsyncSession
8 +
9 +from app.agents.sca.schema.sca import AgentScaOverviewItem
10 +from app.agents.sca.schema.sca import ScaOverviewResponse
11 +from app.agents.sca.schema.sca import ScaStatsResponse
12 +from app.agents.wazuh.services.sca import collect_agent_sca
13 +from app.db.universal_models import Agents
14 +
15 +
16 +async def get_all_agents_from_db(
17 + db_session: AsyncSession,
18 + customer_code: Optional[str] = None,
19 +) -> List[Agents]:
20 + """
21 + Get all agents from database, optionally filtered by customer code
22 +
23 + Args:
24 + db_session: Database session to use
25 + customer_code: Optional customer code to filter agents by
26 +
27 + Returns:
28 + List of Agent objects
29 + """
30 + try:
31 + query = select(Agents)
32 + if customer_code:
33 + query = query.filter(Agents.customer_code == customer_code)
34 +
35 + result = await db_session.execute(query)
36 + agents = result.scalars().all()
37 +
38 + logger.info(f"Found {len(agents)} agents" + (f" for customer {customer_code}" if customer_code else ""))
39 + return agents
40 +
41 + except Exception as e:
42 + logger.error(f"Error fetching agents from database: {e}")
43 + raise HTTPException(status_code=500, detail=f"Failed to fetch agents: {e}")
44 +
45 +
46 +async def collect_sca_for_all_agents(
47 + db_session: AsyncSession,
48 + customer_code: Optional[str] = None,
49 + agent_name: Optional[str] = None,
50 + policy_id: Optional[str] = None,
51 + policy_name: Optional[str] = None,
52 + min_score: Optional[int] = None,
53 + max_score: Optional[int] = None,
54 +) -> List[AgentScaOverviewItem]:
55 + """
56 + Collect SCA results for all agents from Wazuh Manager
57 +
58 + Args:
59 + db_session: Database session to use
60 + customer_code: Optional customer code filter
61 + agent_name: Optional agent name filter
62 + policy_id: Optional policy ID filter
63 + policy_name: Optional policy name filter (partial matching)
64 + min_score: Optional minimum score filter
65 + max_score: Optional maximum score filter
66 +
67 + Returns:
68 + List of AgentScaOverviewItem objects
69 + """
70 + try:
71 + # Get agents from database
72 + agents = await get_all_agents_from_db(db_session, customer_code)
73 +
74 + if not agents:
75 + logger.warning("No agents found" + (f" for customer {customer_code}" if customer_code else ""))
76 + return []
77 +
78 + all_sca_results = []
79 +
80 + for agent in agents:
81 + # Skip if agent name filter is specified and doesn't match
82 + if agent_name and agent.hostname != agent_name:
83 + continue
84 +
85 + try:
86 + logger.info(f"Collecting SCA results for agent: {agent.hostname}")
87 +
88 + # Collect SCA data from Wazuh Manager for this agent
89 + sca_response = await collect_agent_sca(agent.agent_id)
90 +
91 + if not sca_response.success or not sca_response.sca:
92 + logger.warning(f"No SCA data found for agent {agent.hostname}")
93 + continue
94 +
95 + # Process each SCA policy result for this agent
96 + for sca_result in sca_response.sca:
97 + # Apply filters
98 + if policy_id and sca_result.policy_id != policy_id:
99 + continue
100 +
101 + if policy_name and policy_name.lower() not in sca_result.name.lower():
102 + continue
103 +
104 + if min_score is not None and sca_result.score < min_score:
105 + continue
106 +
107 + if max_score is not None and sca_result.score > max_score:
108 + continue
109 +
110 + # Create overview item
111 + overview_item = AgentScaOverviewItem(
112 + agent_id=agent.agent_id,
113 + agent_name=agent.hostname,
114 + customer_code=agent.customer_code,
115 + policy_id=sca_result.policy_id,
116 + policy_name=sca_result.name,
117 + description=sca_result.description,
118 + total_checks=sca_result.total_checks,
119 + pass_count=sca_result.pass_count,
120 + fail_count=sca_result.fail,
121 + invalid_count=sca_result.invalid,
122 + score=sca_result.score,
123 + start_scan=sca_result.start_scan,
124 + end_scan=sca_result.end_scan,
125 + references=sca_result.references,
126 + hash_file=sca_result.hash_file,
127 + )
128 +
129 + all_sca_results.append(overview_item)
130 +
131 + except Exception as e:
132 + logger.error(f"Error collecting SCA for agent {agent.hostname}: {e}")
133 + # Continue with other agents even if one fails
134 + continue
135 +
136 + logger.info(f"Collected SCA results for {len(all_sca_results)} policy results across agents")
137 + return all_sca_results
138 +
139 + except Exception as e:
140 + logger.error(f"Error collecting SCA for all agents: {e}")
141 + raise HTTPException(status_code=500, detail=f"Failed to collect SCA results: {e}")
142 +
143 +
144 +async def search_sca_overview(
145 + db_session: AsyncSession,
146 + customer_code: Optional[str] = None,
147 + agent_name: Optional[str] = None,
148 + policy_id: Optional[str] = None,
149 + policy_name: Optional[str] = None,
150 + min_score: Optional[int] = None,
151 + max_score: Optional[int] = None,
152 + page: int = 1,
153 + page_size: int = 50,
154 +) -> ScaOverviewResponse:
155 + """
156 + Search SCA results across all agents with filtering and pagination
157 +
158 + Args:
159 + db_session: Database session for agent lookup
160 + customer_code: Optional customer code filter
161 + agent_name: Optional agent hostname filter
162 + policy_id: Optional policy ID filter
163 + policy_name: Optional policy name filter (partial matching)
164 + min_score: Optional minimum score filter
165 + max_score: Optional maximum score filter
166 + page: Page number for pagination
167 + page_size: Number of results per page
168 +
169 + Returns:
170 + ScaOverviewResponse with paginated results and statistics
171 + """
172 + logger.info(
173 + f"Searching SCA overview with filters: customer_code={customer_code}, "
174 + f"agent_name={agent_name}, policy_id={policy_id}, policy_name={policy_name}, "
175 + f"min_score={min_score}, max_score={max_score}, page={page}, page_size={page_size}",
176 + )
177 +
178 + # Build filters applied dict for response
179 + filters_applied = {}
180 + if customer_code:
181 + filters_applied["customer_code"] = customer_code
182 + if agent_name:
183 + filters_applied["agent_name"] = agent_name
184 + if policy_id:
185 + filters_applied["policy_id"] = policy_id
186 + if policy_name:
187 + filters_applied["policy_name"] = policy_name
188 + if min_score is not None:
189 + filters_applied["min_score"] = min_score
190 + if max_score is not None:
191 + filters_applied["max_score"] = max_score
192 +
193 + try:
194 + # Collect all SCA results with filtering
195 + all_sca_results = await collect_sca_for_all_agents(
196 + db_session=db_session,
197 + customer_code=customer_code,
198 + agent_name=agent_name,
199 + policy_id=policy_id,
200 + policy_name=policy_name,
201 + min_score=min_score,
202 + max_score=max_score,
203 + )
204 +
205 + total_count = len(all_sca_results)
206 +
207 + # Calculate pagination
208 + total_pages = (total_count + page_size - 1) // page_size
209 + start_idx = (page - 1) * page_size
210 + end_idx = start_idx + page_size
211 +
212 + # Get paginated results
213 + paginated_results = all_sca_results[start_idx:end_idx]
214 +
215 + # Calculate statistics
216 + unique_agents = set(item.agent_id for item in all_sca_results)
217 + unique_policies = set(item.policy_id for item in all_sca_results)
218 +
219 + total_checks_all = sum(item.total_checks for item in all_sca_results)
220 + total_passes_all = sum(item.pass_count for item in all_sca_results)
221 + total_fails_all = sum(item.fail_count for item in all_sca_results)
222 + total_invalid_all = sum(item.invalid_count for item in all_sca_results)
223 +
224 + # Calculate average score
225 + average_score = sum(item.score for item in all_sca_results) / len(all_sca_results) if all_sca_results else 0.0
226 +
227 + return ScaOverviewResponse(
228 + sca_results=paginated_results,
229 + total_count=total_count,
230 + total_agents=len(unique_agents),
231 + total_policies=len(unique_policies),
232 + average_score=round(average_score, 2),
233 + total_checks_all_agents=total_checks_all,
234 + total_passes_all_agents=total_passes_all,
235 + total_fails_all_agents=total_fails_all,
236 + total_invalid_all_agents=total_invalid_all,
237 + page=page,
238 + page_size=page_size,
239 + total_pages=total_pages,
240 + has_next=page < total_pages,
241 + has_previous=page > 1,
242 + success=True,
243 + message=f"Found {total_count} SCA results across {len(unique_agents)} agents",
244 + filters_applied=filters_applied,
245 + )
246 +
247 + except Exception as e:
248 + logger.error(f"Error in SCA overview search: {e}")
249 + raise HTTPException(status_code=500, detail=f"Failed to search SCA results: {e}")
250 +
251 +
252 +async def get_sca_statistics(
253 + db_session: AsyncSession,
254 + customer_code: Optional[str] = None,
255 +) -> ScaStatsResponse:
256 + """
257 + Get SCA statistics across all agents or for a specific customer
258 +
259 + Args:
260 + db_session: Database session to use
261 + customer_code: Optional customer code to filter by
262 +
263 + Returns:
264 + ScaStatsResponse with SCA statistics
265 + """
266 + try:
267 + logger.info("Getting SCA statistics" + (f" for customer {customer_code}" if customer_code else " for all customers"))
268 +
269 + # Collect all SCA results
270 + all_sca_results = await collect_sca_for_all_agents(
271 + db_session=db_session,
272 + customer_code=customer_code,
273 + )
274 +
275 + if not all_sca_results:
276 + return ScaStatsResponse(
277 + total_agents_with_sca=0,
278 + total_policies=0,
279 + average_score_across_all=0.0,
280 + total_checks_all_agents=0,
281 + total_passes_all_agents=0,
282 + total_fails_all_agents=0,
283 + total_invalid_all_agents=0,
284 + by_customer={},
285 + success=True,
286 + message="No SCA results found",
287 + )
288 +
289 + # Calculate overall statistics
290 + unique_agents = set(item.agent_id for item in all_sca_results)
291 + unique_policies = set(item.policy_id for item in all_sca_results)
292 +
293 + total_checks_all = sum(item.total_checks for item in all_sca_results)
294 + total_passes_all = sum(item.pass_count for item in all_sca_results)
295 + total_fails_all = sum(item.fail_count for item in all_sca_results)
296 + total_invalid_all = sum(item.invalid_count for item in all_sca_results)
297 +
298 + average_score = sum(item.score for item in all_sca_results) / len(all_sca_results)
299 +
300 + # Group by customer if no specific customer requested
301 + by_customer = {}
302 + if not customer_code:
303 + customer_groups = {}
304 + for item in all_sca_results:
305 + cust_code = item.customer_code or "unknown"
306 + if cust_code not in customer_groups:
307 + customer_groups[cust_code] = []
308 + customer_groups[cust_code].append(item)
309 +
310 + for cust_code, items in customer_groups.items():
311 + unique_agents_cust = set(item.agent_id for item in items)
312 + unique_policies_cust = set(item.policy_id for item in items)
313 + avg_score_cust = sum(item.score for item in items) / len(items)
314 +
315 + by_customer[cust_code] = {
316 + "total_agents": len(unique_agents_cust),
317 + "total_policies": len(unique_policies_cust),
318 + "average_score": round(avg_score_cust, 2),
319 + "total_checks": sum(item.total_checks for item in items),
320 + "total_passes": sum(item.pass_count for item in items),
321 + "total_fails": sum(item.fail_count for item in items),
322 + "total_invalid": sum(item.invalid_count for item in items),
323 + }
324 +
325 + return ScaStatsResponse(
326 + total_agents_with_sca=len(unique_agents),
327 + total_policies=len(unique_policies),
328 + average_score_across_all=round(average_score, 2),
329 + total_checks_all_agents=total_checks_all,
330 + total_passes_all_agents=total_passes_all,
331 + total_fails_all_agents=total_fails_all,
332 + total_invalid_all_agents=total_invalid_all,
333 + by_customer=by_customer,
334 + success=True,
335 + message=f"SCA statistics calculated for {len(unique_agents)} agents",
336 + )
337 +
338 + except Exception as e:
339 + logger.error(f"Error getting SCA statistics: {e}")
340 + raise HTTPException(status_code=500, detail=f"Failed to get SCA statistics: {e}")
backend/app/routers/agents.py
+2
@@ -1,6 +1,7 @@
1 from fastapi import APIRouter
2
3 from app.agents.routes.agents import agents_router
4 +from app.agents.sca.routes.sca import sca_router
5 from app.agents.vulnerabilities.routes.vulnerabilities import vulnerabilities_router
6
7 # Instantiate the APIRouter
@@ -9,3 +10,4 @@ router = APIRouter()
10 # Include the Wazuh Manager related routes
11 router.include_router(agents_router, prefix="/agents", tags=["agents"])
12 router.include_router(vulnerabilities_router, prefix="/vulnerabilities", tags=["vulnerabilities"])
13 +router.include_router(sca_router, prefix="/sca", tags=["sca"])