@cryptotaxi247 / CoPilot / commits / 099166d4

fix: retry dashboard terms agg with .keyword on text-field errors (#861)

* fix: retry dashboard terms agg with .keyword on text-field errors Dashboard panels of type `pie` / `bar_h` build a terms aggregation on the field name declared in the panel template (e.g. `agent_name`, `data_win_system_eventID`). Elasticsearch refuses terms aggs on `text`-typed fields by default — they lack per-document field data. Customer indices that haven't received the proper Wazuh template (or were created with a Graylog index set whose custom field mappings don't include `.keyword` subfields) hit this: RequestError(400, 'search_phase_execution_exception', 'Text fields are not optimised for operations that require per-document field data like aggregations and sorting... Please use a keyword field instead.') Other indices in the same deployment can have proper keyword mappings and work fine, which is what makes this surprising in the wild — a dashboard breaks for one customer's Wazuh index but not another's. Fix: catch the specific RequestError, append `.keyword` to the field, and retry once. Cheap one-shot fallback. If the second attempt also fails the original error propagates to the existing per-panel error handler so the panel still renders an error card without taking down the whole dashboard. Helper: _is_text_field_agg_error(exc) checks both `exc.error` (the type code, which is `search_phase_execution_exception`) and `exc.info` (the human-readable message, where "Text fields are not optimised" appears). Using both fields means we don't false-trigger on other search_phase_execution_exceptions like "shard 0 failed: index_not_found_exception". Matches behavior is stable across Elasticsearch 7.x. Note: `str(exc)` on elasticsearch7's RequestError only includes the error code, NOT the message body — the human-readable cause lives on `exc.info`. Easy to get wrong; the helper docstring calls it out. Verified with three constructed RequestError cases in the rebuilt container: 1. real text-field agg error -> match=True 2. unrelated parsing_exception (400) -> match=False 3. search_phase_execution but different -> match=False Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix: simplify error handling logic for text field aggregation --------- Co-authored-by: taylor_socfortress <taylor.walton@socfortress.co> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

taylorcopilot committed May 8, 2026 at 14:44 UTC 099166d4a54cac17dd580bc3b1b9f97277891497
1 file changed +44 -8
backend/app/siem/services/dashboards.py
+44 -8
@@ -4,6 +4,7 @@ from typing import Any
4 from typing import Dict
5 from typing import List
6
7 +from elasticsearch7.exceptions import RequestError
8 from fastapi import HTTPException
9 from loguru import logger
10 from sqlalchemy import select
@@ -146,6 +147,25 @@ async def disable_dashboard(
147 # ── Panel data (execute queries for dashboard rendering) ─────────
148
149
150 +def _is_text_field_agg_error(exc: RequestError) -> bool:
151 + """True iff the Elasticsearch error is the one raised when a terms agg
152 + targets a `text`-typed field (which lacks per-document field data by
153 + default). The error string is stable across ES 7.x and is the cue we
154 + use to retry the agg against `<field>.keyword`.
155 +
156 + Sample message body (in `exc.info`):
157 + "Text fields are not optimised for operations that require
158 + per-document field data like aggregations and sorting..."
159 +
160 + Note: `str(exc)` for elasticsearch7 RequestError only includes the
161 + error code ("search_phase_execution_exception"), NOT the message —
162 + the human-readable text lives on `exc.info`. We match against both
163 + so the type code and the specific cause both have to line up.
164 + """
165 + info = getattr(exc, "info", "") or ""
166 + return getattr(exc, "error", "") == "search_phase_execution_exception" and "Text fields are not optimised" in str(info)
167 +
168 +
169 def _compute_histogram_interval(timerange: str) -> str:
170 """Pick a reasonable date_histogram interval for the given timerange."""
171 mapping = {
@@ -258,15 +278,31 @@ async def get_panel_data(
278 if not field:
279 results[pid] = PanelResult(type=ptype, error="No field specified for aggregation")
280 continue
261 - body["aggs"] = {
262 - "top_values": {
263 - "terms": {
264 - "field": field,
265 - "size": size,
281 +
282 + def _build_terms_agg(agg_field: str) -> dict:
283 + return {
284 + "top_values": {
285 + "terms": {"field": agg_field, "size": size},
286 },
267 - },
268 - }
269 - resp = await es_client.search(index=event_source.index_pattern, body=body)
287 + }
288 +
289 + body["aggs"] = _build_terms_agg(field)
290 + try:
291 + resp = await es_client.search(index=event_source.index_pattern, body=body)
292 + except RequestError as exc:
293 + # Elasticsearch refuses terms aggs on `text` fields by default.
294 + # If the index maps `field` as text, retry with the conventional
295 + # `field.keyword` subfield. Cheap one-shot fallback that handles
296 + # the common case where customer index templates differ.
297 + if _is_text_field_agg_error(exc) and not field.endswith(".keyword"):
298 + logger.info(
299 + f"Panel {pid}: '{field}' is text-typed in {event_source.index_pattern}; "
300 + f"retrying with '{field}.keyword'",
301 + )
302 + body["aggs"] = _build_terms_agg(f"{field}.keyword")
303 + resp = await es_client.search(index=event_source.index_pattern, body=body)
304 + else:
305 + raise
306 buckets = resp["aggregations"]["top_values"]["buckets"]
307 labels = [str(b["key"]) for b in buckets]
308 data = [b["doc_count"] for b in buckets]