| 1 | #!/usr/bin/env python3 |
| 2 | """Observability metrics schema helpers for crawl and analysis pipelines.""" |
| 3 | |
| 4 | from __future__ import annotations |
| 5 | |
| 6 | import json |
| 7 | import math |
| 8 | from dataclasses import asdict, dataclass, field, is_dataclass |
| 9 | from pathlib import Path |
| 10 | from typing import Any |
| 11 | |
| 12 | ROOT = Path(__file__).resolve().parent.parent |
| 13 | DEFAULT_OBSERVABILITY_DIR = ROOT / "data" / "metrics" / "observability" |
| 14 | METRICS_SCHEMA_VERSION = "observability_v1" |
| 15 | |
| 16 | |
| 17 | @dataclass(frozen=True, slots=True) |
| 18 | class CrawlMetrics: |
| 19 | duration_seconds: float |
| 20 | api_calls: int |
| 21 | cache_hits: int |
| 22 | cache_misses: int |
| 23 | stale_cache_hits: int |
| 24 | rate_limit_events: int |
| 25 | secondary_rate_limit_hit: bool |
| 26 | source_type: str |
| 27 | duration_p95_seconds: float | None = None |
| 28 | duration_sample_count: int = 0 |
| 29 | |
| 30 | |
| 31 | @dataclass(frozen=True, slots=True) |
| 32 | class MapReduceMetrics: |
| 33 | stage: str |
| 34 | duration_seconds: float |
| 35 | input_tokens: int |
| 36 | output_tokens: int |
| 37 | cost_usd: float |
| 38 | status: str |
| 39 | gate_failure_reasons: list[str] = field(default_factory=list) |
| 40 | |
| 41 | |
| 42 | @dataclass(frozen=True, slots=True) |
| 43 | class AnalysisMetrics: |
| 44 | duration_seconds: float |
| 45 | token_ledger: dict[str, Any] |
| 46 | map_stages: list[MapReduceMetrics] = field(default_factory=list) |
| 47 | reduce_stage: MapReduceMetrics | None = None |
| 48 | |
| 49 | |
| 50 | @dataclass(frozen=True, slots=True) |
| 51 | class ObservabilityLedger: |
| 52 | schema_version: str |
| 53 | run_id: str |
| 54 | week: str |
| 55 | timestamp: str |
| 56 | crawl_metrics: list[CrawlMetrics] = field(default_factory=list) |
| 57 | analysis_metrics: AnalysisMetrics | None = None |
| 58 | environment: dict[str, Any] = field(default_factory=dict) |
| 59 | |
| 60 | |
| 61 | def duration_p95(durations: list[float]) -> float: |
| 62 | """Return a simple p95 duration estimate for a sampled duration series.""" |
| 63 | if not durations: |
| 64 | return 0.0 |
| 65 | ordered = sorted(float(value) for value in durations) |
| 66 | index = max(0, math.ceil(len(ordered) * 0.95) - 1) |
| 67 | return round(ordered[index], 3) |
| 68 | |
| 69 | |
| 70 | def emit_ledger(ledger: ObservabilityLedger | dict[str, Any], output_path: Path) -> Path: |
| 71 | """Write a validated observability ledger JSON artifact.""" |
| 72 | payload = to_serializable(ledger) |
| 73 | errors = validate_ledger(payload) |
| 74 | if errors: |
| 75 | raise ValueError(f"Invalid observability ledger: {', '.join(errors)}") |
| 76 | output_path.parent.mkdir(parents=True, exist_ok=True) |
| 77 | output_path.write_text( |
| 78 | json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False) + "\n", |
| 79 | encoding="utf-8", |
| 80 | ) |
| 81 | return output_path |
| 82 | |
| 83 | |
| 84 | def validate_ledger(data: dict[str, Any]) -> list[str]: |
| 85 | """Return missing or malformed required field paths for an observability ledger.""" |
| 86 | errors: list[str] = [] |
| 87 | if not isinstance(data, dict): |
| 88 | return ["ledger"] |
| 89 | |
| 90 | _require(data, "schema_version", errors) |
| 91 | _require(data, "run_id", errors) |
| 92 | _require(data, "week", errors) |
| 93 | _require(data, "timestamp", errors) |
| 94 | _require(data, "crawl_metrics", errors) |
| 95 | _require(data, "environment", errors) |
| 96 | if "environment" in data and not isinstance(data["environment"], dict): |
| 97 | errors.append("environment: must be a dict") |
| 98 | sv = data.get("schema_version") |
| 99 | if sv is not None and sv != METRICS_SCHEMA_VERSION: |
| 100 | errors.append(f"schema_version: expected '{METRICS_SCHEMA_VERSION}', got '{sv}'") |
| 101 | |
| 102 | crawl_metrics = data.get("crawl_metrics") |
| 103 | if crawl_metrics is not None and not isinstance(crawl_metrics, list): |
| 104 | errors.append("crawl_metrics") |
| 105 | elif isinstance(crawl_metrics, list): |
| 106 | for index, metric in enumerate(crawl_metrics): |
| 107 | if not isinstance(metric, dict): |
| 108 | errors.append(f"crawl_metrics[{index}]") |
| 109 | continue |
| 110 | _validate_required_fields( |
| 111 | metric, |
| 112 | [ |
| 113 | "duration_seconds", |
| 114 | "api_calls", |
| 115 | "cache_hits", |
| 116 | "cache_misses", |
| 117 | "stale_cache_hits", |
| 118 | "rate_limit_events", |
| 119 | "secondary_rate_limit_hit", |
| 120 | "source_type", |
| 121 | ], |
| 122 | f"crawl_metrics[{index}]", |
| 123 | errors, |
| 124 | ) |
| 125 | |
| 126 | analysis_metrics = data.get("analysis_metrics") |
| 127 | if analysis_metrics is not None: |
| 128 | if not isinstance(analysis_metrics, dict): |
| 129 | errors.append("analysis_metrics") |
| 130 | else: |
| 131 | _validate_required_fields( |
| 132 | analysis_metrics, |
| 133 | ["duration_seconds", "token_ledger", "map_stages"], |
| 134 | "analysis_metrics", |
| 135 | errors, |
| 136 | ) |
| 137 | token_ledger = analysis_metrics.get("token_ledger") |
| 138 | if not isinstance(token_ledger, dict): |
| 139 | errors.append("analysis_metrics.token_ledger") |
| 140 | else: |
| 141 | _validate_required_fields( |
| 142 | token_ledger, |
| 143 | ["input_tokens", "output_tokens", "total_tokens"], |
| 144 | "analysis_metrics.token_ledger", |
| 145 | errors, |
| 146 | ) |
| 147 | map_stages = analysis_metrics.get("map_stages") |
| 148 | if not isinstance(map_stages, list): |
| 149 | errors.append("analysis_metrics.map_stages") |
| 150 | else: |
| 151 | for index, metric in enumerate(map_stages): |
| 152 | _validate_stage(metric, f"analysis_metrics.map_stages[{index}]", errors) |
| 153 | reduce_stage = analysis_metrics.get("reduce_stage") |
| 154 | if reduce_stage is not None: |
| 155 | _validate_stage(reduce_stage, "analysis_metrics.reduce_stage", errors) |
| 156 | |
| 157 | return errors |
| 158 | |
| 159 | |
| 160 | def to_serializable(value: Any) -> Any: |
| 161 | """Normalize dataclasses, paths, and containers into JSON-serializable values.""" |
| 162 | if is_dataclass(value): |
| 163 | return to_serializable(asdict(value)) |
| 164 | if isinstance(value, Path): |
| 165 | return value.as_posix() |
| 166 | if isinstance(value, dict): |
| 167 | return {str(key): to_serializable(item) for key, item in value.items()} |
| 168 | if isinstance(value, list): |
| 169 | return [to_serializable(item) for item in value] |
| 170 | if isinstance(value, tuple): |
| 171 | return [to_serializable(item) for item in value] |
| 172 | return value |
| 173 | |
| 174 | |
| 175 | def _require(data: dict[str, Any], field_name: str, errors: list[str]) -> None: |
| 176 | if field_name not in data: |
| 177 | errors.append(field_name) |
| 178 | |
| 179 | |
| 180 | def _validate_required_fields( |
| 181 | payload: dict[str, Any], |
| 182 | field_names: list[str], |
| 183 | prefix: str, |
| 184 | errors: list[str], |
| 185 | ) -> None: |
| 186 | for field_name in field_names: |
| 187 | if field_name not in payload: |
| 188 | errors.append(f"{prefix}.{field_name}") |
| 189 | |
| 190 | |
| 191 | def _validate_stage(stage: Any, prefix: str, errors: list[str]) -> None: |
| 192 | if not isinstance(stage, dict): |
| 193 | errors.append(prefix) |
| 194 | return |
| 195 | _validate_required_fields( |
| 196 | stage, |
| 197 | [ |
| 198 | "stage", |
| 199 | "duration_seconds", |
| 200 | "input_tokens", |
| 201 | "output_tokens", |
| 202 | "cost_usd", |
| 203 | "status", |
| 204 | "gate_failure_reasons", |
| 205 | ], |
| 206 | prefix, |
| 207 | errors, |
| 208 | ) |