965c0bef · cryptotaxi247/CoPilot · siGit Code & Deploy

@@ -880,9 +880,7 @@ async def search_vulnerabilities_from_indexer(

880

881 # Override customer_code based on user permissions

882 if "*" not in accessible_customers:

883 - # User has limited access - filter by their accessible customers

883 if customer_code and customer_code not in accessible_customers:

885 - # User requested a customer they don't have access to

884 return VulnerabilitySearchResponse(

885 vulnerabilities=[],

886 total_count=0,

@@ -899,7 +897,6 @@ async def search_vulnerabilities_from_indexer(

897 message=f"Access denied to customer {customer_code}",

898 filters_applied={},

899 )

902 - # If no customer_code specified or user has access, we'll filter by accessible customers later

900

901 # Build filters applied dict for response

902 filters_applied = {}

@@ -914,14 +911,12 @@ async def search_vulnerabilities_from_indexer(

911 if package_name:

912 filters_applied["package_name"] = package_name

913

917 - # Create Elasticsearch client

914 es_client = None

915 try:

916 # Initialize Elasticsearch client

917 es_client = await create_wazuh_indexer_client_async("Wazuh-Indexer")

918

923 - # Always get all agents to build hostname to customer_code mapping

924 - # This ensures we can always provide customer_code in the response

919 + # Get all agents for customer code mapping

920 all_agents_query = select(Agents)

921 all_agents_result = await db_session.execute(all_agents_query)

922 all_agents = all_agents_result.scalars().all()

@@ -940,7 +935,6 @@ async def search_vulnerabilities_from_indexer(

935

936 # Apply user access restrictions first

937 if "*" not in accessible_customers:

943 - # User has limited access - only show their customers' agents

938 query = query.filter(Agents.customer_code.in_(accessible_customers))

939

940 # Apply additional filters if specified

@@ -971,7 +965,6 @@ async def search_vulnerabilities_from_indexer(

965 )

966

967 # Build list of agent hostnames for Elasticsearch filtering

974 - # If user has restricted access, always filter by their accessible agents

968 if "*" not in accessible_customers or customer_code or agent_name:

969 for agent in agents:

970 if agent.hostname:

@@ -1016,146 +1009,65 @@ async def search_vulnerabilities_from_indexer(

1009 if package_name:

1010 es_query["bool"]["must"].append({"wildcard": {"package.name": f"*{package_name}*"}})

1011

1019 - # Calculate pagination

1020 - start_index = (page - 1) * page_size

1012 + # First, get total count and severity aggregations

1013 + count_response = await es_client.count(index=",".join(vuln_indices), body={"query": es_query})

1014 + total_count = count_response["count"]

1015 +

1016 + # Get severity aggregations

1017 + agg_response = await es_client.search(

1018 + index=",".join(vuln_indices),

1019 + body={

1020 + "query": es_query,

1021 + "size": 0,

1022 + "aggs": {"severity_counts": {"terms": {"field": "vulnerability.severity", "size": 10}}},

1023 + },

1024 + )

1025

1022 - # Create Elasticsearch client

1023 - es_client = None

1024 - try:

1025 - es_client = await create_wazuh_indexer_client_async("Wazuh-Indexer")

1026 -

1027 - # First, get total count and severity aggregations

1028 - count_response = await es_client.count(index=",".join(vuln_indices), body={"query": es_query})

1029 - total_count = count_response["count"]

1030 -

1031 - # Get severity aggregations

1032 - agg_response = await es_client.search(

1033 - index=",".join(vuln_indices),

1034 - body={

1035 - "query": es_query,

1036 - "size": 0, # We don't need documents, just aggregations

1037 - "aggs": {"severity_counts": {"terms": {"field": "vulnerability.severity", "size": 10}}},

1038 - },

1026 + # Extract severity counts

1027 + severity_counts = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0}

1028 + if "aggregations" in agg_response and "severity_counts" in agg_response["aggregations"]:

1029 + for bucket in agg_response["aggregations"]["severity_counts"]["buckets"]:

1030 + severity_value = bucket["key"]

1031 + count = bucket["doc_count"]

1032 + if severity_value in severity_counts:

1033 + severity_counts[severity_value] = count

1034 +

1035 + # Calculate pagination info

1036 + total_pages = (total_count + page_size - 1) // page_size

1037 + has_next = page < total_pages

1038 + has_previous = page > 1

1039 +

1040 + if total_count == 0:

1041 + return VulnerabilitySearchResponse(

1042 + vulnerabilities=[],

1043 + total_count=0,

1044 + critical_count=0,

1045 + high_count=0,

1046 + medium_count=0,

1047 + low_count=0,

1048 + page=page,

1049 + page_size=page_size,

1050 + total_pages=0,

1051 + has_next=False,

1052 + has_previous=False,

1053 + success=True,

1054 + message="No vulnerabilities found matching the specified criteria",

1055 + filters_applied=filters_applied,

1056 )

1057

1041 - # Extract severity counts from aggregation response

1042 - severity_counts = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0}

1043 - if "aggregations" in agg_response and "severity_counts" in agg_response["aggregations"]:

1044 - for bucket in agg_response["aggregations"]["severity_counts"]["buckets"]:

1045 - severity = bucket["key"]

1046 - count = bucket["doc_count"]

1047 - if severity in severity_counts:

1048 - severity_counts[severity] = count

1049 -

1050 - # Calculate pagination info

1051 - total_pages = (total_count + page_size - 1) // page_size

1052 - has_next = page < total_pages

1053 - has_previous = page > 1

1054 -

1055 - if total_count == 0:

1056 - return VulnerabilitySearchResponse(

1057 - vulnerabilities=[],

1058 - total_count=0,

1059 - critical_count=0,

1060 - high_count=0,

1061 - medium_count=0,

1062 - low_count=0,

1063 - page=page,

1064 - page_size=page_size,

1065 - total_pages=0,

1066 - has_next=False,

1067 - has_previous=False,

1068 - success=True,

1069 - message="No vulnerabilities found matching the specified criteria",

1070 - filters_applied=filters_applied,

1071 - )

1058 + # Calculate start index

1059 + start_index = (page - 1) * page_size

1060

1073 - # Get the actual results with pagination

1074 - search_response = await es_client.search(

1075 - index=",".join(vuln_indices),

1076 - body={

1077 - "query": es_query,

1078 - "sort": [{"vulnerability.detected_at": {"order": "desc"}}, {"vulnerability.severity": {"order": "asc"}}],

1079 - "from": start_index,

1080 - "size": page_size,

1081 - },

1061 + # Check if pagination exceeds Elasticsearch's 10,000 result window

1062 + if start_index >= 10000:

1063 + logger.warning(

1064 + f"Deep pagination requested (page {page}, start_index {start_index}). "

1065 + f"Elasticsearch limits pagination to 10,000 results. "

1066 + f"Please use more specific filters or export to CSV report.",

1067 )

1068

1084 - vulnerabilities = []

1085 - for hit in search_response["hits"]["hits"]:

1086 - try:

1087 - source = hit["_source"]

1088 - agent_data = source.get("agent", {})

1089 - agent_hostname = agent_data.get("name", "unknown")

1090 -

1091 - # Get customer code from our mapping

1092 - agent_customer_code = customer_agent_map.get(agent_hostname)

1093 -

1094 - # Process the vulnerability data

1095 - vuln_data = process_wazuh_document(hit)

1096 -

1097 - # Get EPSS score for the CVE (if requested)

1098 - epss_score, epss_percentile = None, None

1099 - if include_epss:

1100 - epss_score, epss_percentile = await get_epss_score_for_cve(vuln_data.cve_id)

1101 -

1102 - vulnerability_item = VulnerabilitySearchItem(

1103 - cve_id=vuln_data.cve_id,

1104 - severity=vuln_data.severity,

1105 - title=vuln_data.title,

1106 - agent_name=agent_hostname,

1107 - customer_code=agent_customer_code,

1108 - references=vuln_data.references,

1109 - detected_at=vuln_data.detected_at,

1110 - published_at=vuln_data.published_at,

1111 - base_score=vuln_data.base_score,

1112 - package_name=vuln_data.package_name,

1113 - package_version=vuln_data.package_version,

1114 - package_architecture=vuln_data.package_architecture,

1115 - epss_score=epss_score,

1116 - epss_percentile=epss_percentile,

1117 - )

1118 - vulnerabilities.append(vulnerability_item)

1119 -

1120 - except Exception as e:

1121 - logger.error(f"Error processing vulnerability document: {e}")

1122 - continue

1123 -

1124 - # Sort vulnerabilities by EPSS score (highest to lowest) if EPSS is included

1125 - if include_epss:

1126 - # Sort by EPSS score descending, treating None/null as 0

1127 - # Then by severity (Critical=0, High=1, Medium=2, Low=3) for tie-breaking

1128 - severity_order = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3}

1129 -

1130 - def get_epss_sort_key(vuln):

1131 - # Convert EPSS score to float for sorting, handle string/None values

1132 - epss_score = vuln.epss_score

1133 - if epss_score is None:

1134 - epss_float = 0.0

1135 - else:

1136 - try:

1137 - epss_float = float(epss_score)

1138 - except (ValueError, TypeError):

1139 - epss_float = 0.0

1140 - return (

1141 - -epss_float, # Negative for descending order

1142 - severity_order.get(vuln.severity, 4), # Secondary sort by severity

1143 - vuln.cve_id, # Tertiary sort by CVE ID for consistency

1144 - )

1145 -

1146 - vulnerabilities.sort(key=get_epss_sort_key)

1147 - logger.info(f"Sorted {len(vulnerabilities)} vulnerabilities by EPSS score (highest to lowest)")

1148 -

1149 - message = f"Found {len(vulnerabilities)} vulnerabilities on page {page} of {total_pages}"

1150 - if filters_applied:

1151 - message += f" with filters: {filters_applied}"

1152 - if include_epss:

1153 - message += " (sorted by EPSS score, highest to lowest)"

1154 - else:

1155 - message += " (sorted by detection date and severity)"

1156 -

1069 return VulnerabilitySearchResponse(

1158 - vulnerabilities=vulnerabilities,

1070 + vulnerabilities=[],

1071 total_count=total_count,

1072 critical_count=severity_counts["Critical"],

1073 high_count=severity_counts["High"],

@@ -1166,39 +1078,127 @@ async def search_vulnerabilities_from_indexer(

1078 total_pages=total_pages,

1079 has_next=has_next,

1080 has_previous=has_previous,

1169 - success=True,

1170 - message=message,

1081 + success=False,

1082 + message=(

1083 + f"Deep pagination not supported beyond 10,000 results (requested page {page}, position {start_index}). "

1084 + "Please use more specific filters to narrow down results or use the CSV export feature "

1085 + "for accessing all vulnerabilities. Maximum supported page: 200 (with page_size=50)."

1086 + ),

1087 filters_applied=filters_applied,

1088 )

1089

1174 - except Exception as e:

1175 - logger.error(f"Error searching vulnerabilities from indexer: {e}")

1176 - return VulnerabilitySearchResponse(

1177 - vulnerabilities=[],

1178 - total_count=0,

1179 - critical_count=0,

1180 - high_count=0,

1181 - medium_count=0,

1182 - low_count=0,

1183 - page=page,

1184 - page_size=page_size,

1185 - total_pages=0,

1186 - has_next=False,

1187 - has_previous=False,

1188 - success=False,

1189 - message=f"Failed to search vulnerabilities: {e}",

1190 - filters_applied=filters_applied if "filters_applied" in locals() else {},

1090 + # Use standard pagination (within Elasticsearch limits)

1091 + search_response = await es_client.search(

1092 + index=",".join(vuln_indices),

1093 + body={

1094 + "query": es_query,

1095 + "sort": [

1096 + {"vulnerability.detected_at": {"order": "desc"}},

1097 + {"vulnerability.severity": {"order": "asc"}},

1098 + {"_id": {"order": "asc"}}, # Tiebreaker for consistent sorting

1099 + ],

1100 + "from": start_index,

1101 + "size": page_size,

1102 + },

1103 + )

1104 + hits = search_response["hits"]["hits"]

1105 +

1106 + # Process the results

1107 + vulnerabilities = []

1108 + for hit in hits:

1109 + try:

1110 + source = hit["_source"]

1111 + agent_data = source.get("agent", {})

1112 + agent_hostname = agent_data.get("name", "unknown")

1113 +

1114 + # Get customer code from our mapping

1115 + agent_customer_code = customer_agent_map.get(agent_hostname)

1116 +

1117 + # Process the vulnerability data

1118 + vuln_data = process_wazuh_document(hit)

1119 +

1120 + # Get EPSS score for the CVE (if requested)

1121 + epss_score, epss_percentile = None, None

1122 + if include_epss:

1123 + epss_score, epss_percentile = await get_epss_score_for_cve(vuln_data.cve_id)

1124 +

1125 + vulnerability_item = VulnerabilitySearchItem(

1126 + cve_id=vuln_data.cve_id,

1127 + severity=vuln_data.severity,

1128 + title=vuln_data.title,

1129 + agent_name=agent_hostname,

1130 + customer_code=agent_customer_code,

1131 + references=vuln_data.references,

1132 + detected_at=vuln_data.detected_at,

1133 + published_at=vuln_data.published_at,

1134 + base_score=vuln_data.base_score,

1135 + package_name=vuln_data.package_name,

1136 + package_version=vuln_data.package_version,

1137 + package_architecture=vuln_data.package_architecture,

1138 + epss_score=epss_score,

1139 + epss_percentile=epss_percentile,

1140 + )

1141 + vulnerabilities.append(vulnerability_item)

1142 +

1143 + except Exception as e:

1144 + logger.error(f"Error processing vulnerability document: {e}")

1145 + continue

1146 +

1147 + # Sort vulnerabilities by EPSS score if included

1148 + if include_epss:

1149 + severity_order = {"Critical": 0, "High": 1, "Medium": 2, "Low": 3}

1150 +

1151 + def get_epss_sort_key(vuln):

1152 + epss_score_val = vuln.epss_score

1153 + if epss_score_val is None:

1154 + epss_float = 0.0

1155 + else:

1156 + try:

1157 + epss_float = float(epss_score_val)

1158 + except (ValueError, TypeError):

1159 + epss_float = 0.0

1160 + return (

1161 + -epss_float,

1162 + severity_order.get(vuln.severity, 4),

1163 + vuln.cve_id,

1164 + )

1165 +

1166 + vulnerabilities.sort(key=get_epss_sort_key)

1167 + logger.info(f"Sorted {len(vulnerabilities)} vulnerabilities by EPSS score (highest to lowest)")

1168 +

1169 + message = f"Found {len(vulnerabilities)} vulnerabilities on page {page} of {total_pages}"

1170 + if filters_applied:

1171 + message += f" with filters: {filters_applied}"

1172 + if include_epss:

1173 + message += " (sorted by EPSS score, highest to lowest)"

1174 + else:

1175 + message += " (sorted by detection date and severity)"

1176 +

1177 + # Add helpful message when approaching the limit

1178 + if start_index + page_size > 9000:

1179 + message += (

1180 + ". Note: Approaching pagination limit (10,000 results). Consider using filters or CSV export for complete data access."

1181 )

1192 - finally:

1193 - # Ensure the Elasticsearch client session is properly closed

1194 - if es_client:

1195 - try:

1196 - await es_client.close()

1197 - except Exception as close_error:

1198 - logger.warning(f"Error closing Elasticsearch client: {close_error}")

1182 +

1183 + return VulnerabilitySearchResponse(

1184 + vulnerabilities=vulnerabilities,

1185 + total_count=total_count,

1186 + critical_count=severity_counts["Critical"],

1187 + high_count=severity_counts["High"],

1188 + medium_count=severity_counts["Medium"],

1189 + low_count=severity_counts["Low"],

1190 + page=page,

1191 + page_size=page_size,

1192 + total_pages=total_pages,

1193 + has_next=has_next,

1194 + has_previous=has_previous,

1195 + success=True,

1196 + message=message,

1197 + filters_applied=filters_applied,

1198 + )

1199

1200 except Exception as e:

1201 - logger.error(f"Unexpected error in search_vulnerabilities_from_indexer: {e}")

1201 + logger.error(f"Error searching vulnerabilities from indexer: {e}")

1202 return VulnerabilitySearchResponse(

1203 vulnerabilities=[],

1204 total_count=0,

@@ -1212,9 +1212,261 @@ async def search_vulnerabilities_from_indexer(

1212 has_next=False,

1213 has_previous=False,

1214 success=False,

1215 - message=f"Unexpected error occurred: {e}",

1215 + message=f"Failed to search vulnerabilities: {e}",

1216 filters_applied=filters_applied if "filters_applied" in locals() else {},

1217 )

1218 + finally:

1219 + if es_client:

1220 + try:

1221 + await es_client.close()

1222 + except Exception as close_error:

1223 + logger.warning(f"Error closing Elasticsearch client: {close_error}")

1224 +

1225 +

1226 +async def fetch_all_vulnerabilities_for_export(

1227 + db_session: AsyncSession,

1228 + current_user: User,

1229 + customer_code: str,

1230 + agent_name: Optional[str] = None,

1231 + severity: Optional[str] = None,

1232 + cve_id: Optional[str] = None,

1233 + package_name: Optional[str] = None,

1234 + include_epss: bool = True,

1235 +) -> List[VulnerabilitySearchItem]:

1236 + """

1237 + Fetch ALL vulnerabilities for CSV export using scroll API (no pagination limits).

1238 +

1239 + This function is specifically designed for report generation and can handle

1240 + unlimited result sets by using Elasticsearch's scroll API.

1241 +

1242 + Args:

1243 + db_session: Database session for agent lookup

1244 + current_user: Current authenticated user for customer access filtering

1245 + customer_code: Customer code to filter by

1246 + agent_name: Optional agent hostname filter

1247 + severity: Optional severity filter

1248 + cve_id: Optional CVE ID filter

1249 + package_name: Optional package name filter

1250 + include_epss: Whether to include EPSS scores

1251 +

1252 + Returns:

1253 + List of all matching vulnerabilities (no pagination)

1254 + """

1255 + logger.info(

1256 + f"Fetching ALL vulnerabilities for export with filters: customer_code={customer_code}, "

1257 + f"agent_name={agent_name}, severity={severity}, cve_id={cve_id}, "

1258 + f"package_name={package_name}, include_epss={include_epss}",

1259 + )

1260 +

1261 + # Apply customer access filtering

1262 + accessible_customers = await customer_access_handler.get_user_accessible_customers(current_user, db_session)

1263 +

1264 + if "*" not in accessible_customers and customer_code not in accessible_customers:

1265 + logger.warning(f"User {current_user.username} denied access to customer {customer_code}")

1266 + return []

1267 +

1268 + es_client = None

1269 + scroll_id = None

1270 +

1271 + try:

1272 + # Initialize Elasticsearch client

1273 + es_client = await create_wazuh_indexer_client_async("Wazuh-Indexer")

1274 +

1275 + # Get all agents for customer code mapping

1276 + all_agents_query = select(Agents)

1277 + all_agents_result = await db_session.execute(all_agents_query)

1278 + all_agents = all_agents_result.scalars().all()

1279 +

1280 + # Build complete agent hostname to customer code mapping

1281 + customer_agent_map = {}

1282 + for agent in all_agents:

1283 + if agent.hostname:

1284 + customer_agent_map[agent.hostname] = agent.customer_code

1285 +

1286 + # Get agent information for filtering

1287 + agent_hostnames = []

1288 + query = select(Agents).filter(Agents.customer_code == customer_code)

1289 +

1290 + if agent_name:

1291 + query = query.filter(Agents.hostname == agent_name)

1292 +

1293 + result = await db_session.execute(query)

1294 + agents = result.scalars().all()

1295 +

1296 + if not agents:

1297 + logger.warning(f"No agents found for customer {customer_code}")

1298 + return []

1299 +

1300 + for agent in agents:

1301 + if agent.hostname:

1302 + agent_hostnames.append(agent.hostname)

1303 +

1304 + # Get vulnerability indices

1305 + vuln_indices = await get_vulnerabilities_indices()

1306 + if not vuln_indices:

1307 + logger.warning("No vulnerability indices found")

1308 + return []

1309 +

1310 + # Build Elasticsearch query

1311 + es_query = {"bool": {"must": []}}

1312 +

1313 + # Add agent filter

1314 + if agent_hostnames:

1315 + es_query["bool"]["must"].append({"terms": {"agent.name": agent_hostnames}})

1316 +

1317 + # Add severity filter

1318 + if severity:

1319 + es_query["bool"]["must"].append({"term": {"vulnerability.severity": severity}})

1320 +

1321 + # Add CVE ID filter

1322 + if cve_id:

1323 + es_query["bool"]["must"].append({"term": {"vulnerability.id": cve_id}})

1324 +

1325 + # Add package name filter

1326 + if package_name:

1327 + es_query["bool"]["must"].append({"wildcard": {"package.name": f"*{package_name}*"}})

1328 +

1329 + # Use scroll API for unlimited results

1330 + all_vulnerabilities = []

1331 + scroll_size = 1000 # Process 1000 at a time

1332 +

1333 + logger.info(f"Starting scroll search across {len(vuln_indices)} indices")

1334 +

1335 + # Initial scroll request

1336 + scroll_response = await es_client.search(

1337 + index=",".join(vuln_indices),

1338 + body={

1339 + "query": es_query,

1340 + "sort": [

1341 + {"vulnerability.detected_at": {"order": "desc"}},

1342 + {"vulnerability.severity": {"order": "asc"}},

1343 + {"_id": {"order": "asc"}},

1344 + ],

1345 + },

1346 + scroll="5m", # Keep scroll context alive for 5 minutes

1347 + size=scroll_size,

1348 + )

1349 +

1350 + scroll_id = scroll_response["_scroll_id"]

1351 + hits = scroll_response["hits"]["hits"]

1352 +

1353 + logger.info(f"Initial scroll batch: {len(hits)} results")

1354 +

1355 + # Process initial batch

1356 + for hit in hits:

1357 + try:

1358 + source = hit["_source"]

1359 + agent_data = source.get("agent", {})

1360 + agent_hostname = agent_data.get("name", "unknown")

1361 + agent_customer_code = customer_agent_map.get(agent_hostname)

1362 +

1363 + vuln_data = process_wazuh_document(hit)

1364 +

1365 + # Get EPSS score if requested

1366 + epss_score, epss_percentile = None, None

1367 + if include_epss:

1368 + epss_score, epss_percentile = await get_epss_score_for_cve(vuln_data.cve_id)

1369 +

1370 + vulnerability_item = VulnerabilitySearchItem(

1371 + cve_id=vuln_data.cve_id,

1372 + severity=vuln_data.severity,

1373 + title=vuln_data.title,

1374 + agent_name=agent_hostname,

1375 + customer_code=agent_customer_code,

1376 + references=vuln_data.references,

1377 + detected_at=vuln_data.detected_at,

1378 + published_at=vuln_data.published_at,

1379 + base_score=vuln_data.base_score,

1380 + package_name=vuln_data.package_name,

1381 + package_version=vuln_data.package_version,

1382 + package_architecture=vuln_data.package_architecture,

1383 + epss_score=epss_score,

1384 + epss_percentile=epss_percentile,

1385 + )

1386 + all_vulnerabilities.append(vulnerability_item)

1387 +

1388 + except Exception as e:

1389 + logger.error(f"Error processing vulnerability document: {e}")

1390 + continue

1391 +

1392 + # Continue scrolling through all results

1393 + scroll_count = 1

1394 + while len(hits) > 0:

1395 + try:

1396 + scroll_response = await es_client.scroll(scroll_id=scroll_id, scroll="5m")

1397 + scroll_id = scroll_response["_scroll_id"]

1398 + hits = scroll_response["hits"]["hits"]

1399 +

1400 + if not hits:

1401 + break

1402 +

1403 + scroll_count += 1

1404 + logger.info(f"Scroll batch {scroll_count}: {len(hits)} results (total so far: {len(all_vulnerabilities)})")

1405 +

1406 + # Process batch

1407 + for hit in hits:

1408 + try:

1409 + source = hit["_source"]

1410 + agent_data = source.get("agent", {})

1411 + agent_hostname = agent_data.get("name", "unknown")

1412 + agent_customer_code = customer_agent_map.get(agent_hostname)

1413 +

1414 + vuln_data = process_wazuh_document(hit)

1415 +

1416 + # Get EPSS score if requested

1417 + epss_score, epss_percentile = None, None

1418 + if include_epss:

1419 + epss_score, epss_percentile = await get_epss_score_for_cve(vuln_data.cve_id)

1420 +

1421 + vulnerability_item = VulnerabilitySearchItem(

1422 + cve_id=vuln_data.cve_id,

1423 + severity=vuln_data.severity,

1424 + title=vuln_data.title,

1425 + agent_name=agent_hostname,

1426 + customer_code=agent_customer_code,

1427 + references=vuln_data.references,

1428 + detected_at=vuln_data.detected_at,

1429 + published_at=vuln_data.published_at,

1430 + base_score=vuln_data.base_score,

1431 + package_name=vuln_data.package_name,

1432 + package_version=vuln_data.package_version,

1433 + package_architecture=vuln_data.package_architecture,

1434 + epss_score=epss_score,

1435 + epss_percentile=epss_percentile,

1436 + )

1437 + all_vulnerabilities.append(vulnerability_item)

1438 +

1439 + except Exception as e:

1440 + logger.error(f"Error processing vulnerability document: {e}")

1441 + continue

1442 +

1443 + except Exception as scroll_error:

1444 + logger.error(f"Error during scroll: {scroll_error}")

1445 + break

1446 +

1447 + logger.info(f"Successfully fetched {len(all_vulnerabilities)} total vulnerabilities using scroll API")

1448 +

1449 + return all_vulnerabilities

1450 +

1451 + except Exception as e:

1452 + logger.error(f"Error fetching vulnerabilities for export: {e}")

1453 + raise

1454 +

1455 + finally:

1456 + # Always clear the scroll context

1457 + if scroll_id and es_client:

1458 + try:

1459 + await es_client.clear_scroll(scroll_id=scroll_id)

1460 + logger.info("Cleared scroll context")

1461 + except Exception as clear_error:

1462 + logger.warning(f"Could not clear scroll context: {clear_error}")

1463 +

1464 + # Close ES client

1465 + if es_client:

1466 + try:

1467 + await es_client.close()

1468 + except Exception as close_error:

1469 + logger.warning(f"Error closing Elasticsearch client: {close_error}")

1470

1471

1472 async def generate_vulnerability_csv_report(

@@ -1279,50 +1531,36 @@ async def generate_vulnerability_csv_report(

1531

1532 logger.info(f"Generating vulnerability report for customer: {request.customer_code}")

1533

1282 - # Fetch ALL vulnerabilities (no pagination)

1283 - all_vulnerabilities = []

1284 - page = 1

1285 - page_size = 1000 # Large page size for efficiency

1286 -

1287 - while True:

1288 - search_result = await search_vulnerabilities_from_indexer(

1289 - db_session=db_session,

1290 - current_user=current_user,

1291 - customer_code=request.customer_code,

1292 - agent_name=request.agent_name,

1293 - severity=request.severity,

1294 - cve_id=request.cve_id,

1295 - package_name=request.package_name,

1296 - page=page,

1297 - page_size=page_size,

1298 - include_epss=request.include_epss,

1299 - )

1300 -

1301 - if not search_result.success:

1302 - # If we have a report_id, update it to failed status

1303 - if report_id:

1304 - stmt = select(VulnerabilityReport).filter(VulnerabilityReport.id == report_id)

1305 - result = await db_session.execute(stmt)

1306 - report = result.scalars().first()

1307 - if report:

1308 - report.status = "failed"

1309 - report.error_message = search_result.message

1310 - await db_session.commit()

1311 -

1312 - return VulnerabilityReportGenerateResponse(

1313 - success=False,

1314 - message="Failed to fetch vulnerability data",

1315 - error=search_result.message,

1316 - )

1317 -

1318 - all_vulnerabilities.extend(search_result.vulnerabilities)

1534 + # Fetch ALL vulnerabilities using scroll API (no pagination limits)

1535 + all_vulnerabilities = await fetch_all_vulnerabilities_for_export(

1536 + db_session=db_session,

1537 + current_user=current_user,

1538 + customer_code=request.customer_code,

1539 + agent_name=request.agent_name,

1540 + severity=request.severity,

1541 + cve_id=request.cve_id,

1542 + package_name=request.package_name,

1543 + include_epss=request.include_epss,

1544 + )

1545

1320 - if not search_result.has_next:

1321 - break

1546 + logger.info(f"Fetched {len(all_vulnerabilities)} vulnerabilities for report")

1547

1323 - page += 1

1548 + if not all_vulnerabilities:

1549 + # If we have a report_id, update it to failed status

1550 + if report_id:

1551 + stmt = select(VulnerabilityReport).filter(VulnerabilityReport.id == report_id)

1552 + result = await db_session.execute(stmt)

1553 + report = result.scalars().first()

1554 + if report:

1555 + report.status = "failed"

1556 + report.error_message = "No vulnerabilities found matching criteria"

1557 + await db_session.commit()

1558

1325 - logger.info(f"Fetched {len(all_vulnerabilities)} vulnerabilities for report")

1559 + return VulnerabilityReportGenerateResponse(

1560 + success=False,

1561 + message="No vulnerabilities found matching the specified criteria",

1562 + error="No data to export",

1563 + )

1564

1565 # Generate CSV content

1566 csv_buffer = io.StringIO()

581 vulnerability scrolling (#582)