| 1 | #!/usr/bin/env python3 |
| 2 | """Baseline telemetry collection and reporting for crawl matrix readiness. |
| 3 | |
| 4 | Records p50/p95 timings across pipeline stages: |
| 5 | - GitHub crawl |
| 6 | - RSS/news crawl |
| 7 | - Correlation / press context rendering |
| 8 | - Analysis (map/reduce dry-run) |
| 9 | |
| 10 | The baseline window requires at least 5 representative runs (or explicit |
| 11 | rationale for fewer). Telemetry is read from the observability ledger |
| 12 | artifacts in data/metrics/observability/. |
| 13 | |
| 14 | Usage: |
| 15 | python -m scripts.baseline_telemetry report |
| 16 | python -m scripts.baseline_telemetry check --min-runs 5 |
| 17 | |
| 18 | References: |
| 19 | - Issue #333: Define crawl matrix readiness and fan-in validation path |
| 20 | - scripts/observability_metrics.py: Ledger schema |
| 21 | """ |
| 22 | |
| 23 | from __future__ import annotations |
| 24 | |
| 25 | import argparse |
| 26 | import json |
| 27 | import math |
| 28 | import sys |
| 29 | from dataclasses import asdict, dataclass |
| 30 | from pathlib import Path |
| 31 | from typing import Any |
| 32 | |
| 33 | ROOT = Path(__file__).resolve().parent.parent |
| 34 | DEFAULT_METRICS_DIR = ROOT / "data" / "metrics" / "observability" |
| 35 | |
| 36 | # Minimum number of runs for a valid baseline |
| 37 | MINIMUM_BASELINE_RUNS = 5 |
| 38 | |
| 39 | |
| 40 | @dataclass(slots=True) |
| 41 | class StageBaseline: |
| 42 | """Timing baseline for a single pipeline stage.""" |
| 43 | |
| 44 | stage: str |
| 45 | sample_count: int |
| 46 | p50_seconds: float |
| 47 | p95_seconds: float |
| 48 | min_seconds: float |
| 49 | max_seconds: float |
| 50 | mean_seconds: float |
| 51 | |
| 52 | |
| 53 | @dataclass(slots=True) |
| 54 | class BaselineReport: |
| 55 | """Complete baseline telemetry report across all stages.""" |
| 56 | |
| 57 | total_runs: int |
| 58 | observation_window_start: str |
| 59 | observation_window_end: str |
| 60 | stages: list[StageBaseline] |
| 61 | sufficient: bool |
| 62 | rationale: str = "" |
| 63 | |
| 64 | def to_dict(self) -> dict[str, Any]: |
| 65 | return { |
| 66 | "total_runs": self.total_runs, |
| 67 | "observation_window_start": self.observation_window_start, |
| 68 | "observation_window_end": self.observation_window_end, |
| 69 | "stages": [asdict(s) for s in self.stages], |
| 70 | "sufficient": self.sufficient, |
| 71 | "rationale": self.rationale, |
| 72 | } |
| 73 | |
| 74 | |
| 75 | def percentile(values: list[float], pct: float) -> float: |
| 76 | """Compute percentile from sorted values.""" |
| 77 | if not values: |
| 78 | return 0.0 |
| 79 | ordered = sorted(values) |
| 80 | index = max(0, math.ceil(len(ordered) * pct / 100.0) - 1) |
| 81 | return round(ordered[index], 3) |
| 82 | |
| 83 | |
| 84 | def load_ledger_entries(metrics_dir: Path) -> list[dict[str, Any]]: |
| 85 | """Load all observability ledger JSON files from the metrics directory.""" |
| 86 | entries = [] |
| 87 | if not metrics_dir.exists(): |
| 88 | return entries |
| 89 | |
| 90 | for f in sorted(metrics_dir.glob("*.json")): |
| 91 | try: |
| 92 | data = json.loads(f.read_text(encoding="utf-8")) |
| 93 | if isinstance(data, dict) and "crawl_metrics" in data: |
| 94 | entries.append(data) |
| 95 | except (json.JSONDecodeError, OSError): |
| 96 | continue |
| 97 | |
| 98 | return entries |
| 99 | |
| 100 | |
| 101 | def compute_stage_baseline(stage: str, durations: list[float]) -> StageBaseline: |
| 102 | """Compute baseline statistics for a pipeline stage.""" |
| 103 | if not durations: |
| 104 | return StageBaseline( |
| 105 | stage=stage, |
| 106 | sample_count=0, |
| 107 | p50_seconds=0.0, |
| 108 | p95_seconds=0.0, |
| 109 | min_seconds=0.0, |
| 110 | max_seconds=0.0, |
| 111 | mean_seconds=0.0, |
| 112 | ) |
| 113 | |
| 114 | return StageBaseline( |
| 115 | stage=stage, |
| 116 | sample_count=len(durations), |
| 117 | p50_seconds=percentile(durations, 50), |
| 118 | p95_seconds=percentile(durations, 95), |
| 119 | min_seconds=round(min(durations), 3), |
| 120 | max_seconds=round(max(durations), 3), |
| 121 | mean_seconds=round(sum(durations) / len(durations), 3), |
| 122 | ) |
| 123 | |
| 124 | |
| 125 | def build_baseline_report( |
| 126 | entries: list[dict[str, Any]], |
| 127 | min_runs: int = MINIMUM_BASELINE_RUNS, |
| 128 | rationale: str = "", |
| 129 | ) -> BaselineReport: |
| 130 | """Build a baseline telemetry report from observability ledger entries.""" |
| 131 | github_durations: list[float] = [] |
| 132 | rss_durations: list[float] = [] |
| 133 | correlation_durations: list[float] = [] |
| 134 | analysis_durations: list[float] = [] |
| 135 | timestamps: list[str] = [] |
| 136 | |
| 137 | for entry in entries: |
| 138 | ts = entry.get("timestamp", "") |
| 139 | if ts: |
| 140 | timestamps.append(ts) |
| 141 | |
| 142 | crawl_metrics = entry.get("crawl_metrics", []) |
| 143 | for cm in crawl_metrics: |
| 144 | if not isinstance(cm, dict): |
| 145 | continue |
| 146 | duration = cm.get("duration_seconds", 0) |
| 147 | source_type = cm.get("source_type", "") |
| 148 | if source_type == "github": |
| 149 | github_durations.append(float(duration)) |
| 150 | elif source_type in ("rss", "external_news", "techcrunch"): |
| 151 | rss_durations.append(float(duration)) |
| 152 | elif source_type in ("correlation", "press_context"): |
| 153 | correlation_durations.append(float(duration)) |
| 154 | |
| 155 | analysis = entry.get("analysis_metrics") |
| 156 | if isinstance(analysis, dict): |
| 157 | ad = analysis.get("duration_seconds", 0) |
| 158 | if ad: |
| 159 | analysis_durations.append(float(ad)) |
| 160 | |
| 161 | total_runs = len(entries) |
| 162 | window_start = min(timestamps) if timestamps else "" |
| 163 | window_end = max(timestamps) if timestamps else "" |
| 164 | |
| 165 | stages = [ |
| 166 | compute_stage_baseline("github_crawl", github_durations), |
| 167 | compute_stage_baseline("rss_news_crawl", rss_durations), |
| 168 | compute_stage_baseline("correlation_press_context", correlation_durations), |
| 169 | compute_stage_baseline("analysis", analysis_durations), |
| 170 | ] |
| 171 | |
| 172 | sufficient = total_runs >= min_runs |
| 173 | |
| 174 | return BaselineReport( |
| 175 | total_runs=total_runs, |
| 176 | observation_window_start=window_start, |
| 177 | observation_window_end=window_end, |
| 178 | stages=stages, |
| 179 | sufficient=sufficient, |
| 180 | rationale=rationale, |
| 181 | ) |
| 182 | |
| 183 | |
| 184 | def check_trigger_thresholds(report: BaselineReport) -> dict[str, Any]: |
| 185 | """Evaluate trigger thresholds against baseline telemetry. |
| 186 | |
| 187 | Returns a dict with threshold status for each experiment gate. |
| 188 | """ |
| 189 | rss_stage = next((s for s in report.stages if s.stage == "rss_news_crawl"), None) |
| 190 | github_stage = next((s for s in report.stages if s.stage == "github_crawl"), None) |
| 191 | |
| 192 | return { |
| 193 | "rss_matrix_triggers": { |
| 194 | "p95_exceeds_60s": rss_stage.p95_seconds > 60.0 if rss_stage else False, |
| 195 | "p95_value": rss_stage.p95_seconds if rss_stage else 0.0, |
| 196 | "threshold": 60.0, |
| 197 | "triggered": (rss_stage.p95_seconds > 60.0) if rss_stage else False, |
| 198 | }, |
| 199 | "github_shard_triggers": { |
| 200 | "baseline_p95": github_stage.p95_seconds if github_stage else 0.0, |
| 201 | "speedup_threshold_pct": 25.0, |
| 202 | "api_growth_ceiling_pct": 10.0, |
| 203 | "secondary_rate_limit_regression": False, |
| 204 | "triggered": False, # Requires experiment comparison |
| 205 | }, |
| 206 | "baseline_sufficient": report.sufficient, |
| 207 | "total_runs": report.total_runs, |
| 208 | "minimum_required": MINIMUM_BASELINE_RUNS, |
| 209 | } |
| 210 | |
| 211 | |
| 212 | def main() -> int: |
| 213 | parser = argparse.ArgumentParser(description="Crawl matrix baseline telemetry") |
| 214 | sub = parser.add_subparsers(dest="command") |
| 215 | |
| 216 | report_cmd = sub.add_parser("report", help="Generate baseline telemetry report") |
| 217 | report_cmd.add_argument( |
| 218 | "--metrics-dir", |
| 219 | type=Path, |
| 220 | default=DEFAULT_METRICS_DIR, |
| 221 | help="Path to observability metrics directory", |
| 222 | ) |
| 223 | report_cmd.add_argument("--output", type=Path, help="Write report JSON to file") |
| 224 | |
| 225 | check_cmd = sub.add_parser("check", help="Check baseline readiness") |
| 226 | check_cmd.add_argument( |
| 227 | "--metrics-dir", |
| 228 | type=Path, |
| 229 | default=DEFAULT_METRICS_DIR, |
| 230 | help="Path to observability metrics directory", |
| 231 | ) |
| 232 | check_cmd.add_argument( |
| 233 | "--min-runs", |
| 234 | type=int, |
| 235 | default=MINIMUM_BASELINE_RUNS, |
| 236 | help="Minimum number of runs required", |
| 237 | ) |
| 238 | |
| 239 | args = parser.parse_args() |
| 240 | |
| 241 | if args.command == "report": |
| 242 | entries = load_ledger_entries(args.metrics_dir) |
| 243 | report = build_baseline_report(entries) |
| 244 | output = json.dumps(report.to_dict(), indent=2) |
| 245 | if args.output: |
| 246 | args.output.parent.mkdir(parents=True, exist_ok=True) |
| 247 | args.output.write_text(output + "\n", encoding="utf-8") |
| 248 | print(f"Report written to {args.output}") |
| 249 | else: |
| 250 | print(output) |
| 251 | return 0 |
| 252 | |
| 253 | elif args.command == "check": |
| 254 | entries = load_ledger_entries(args.metrics_dir) |
| 255 | report = build_baseline_report(entries, min_runs=args.min_runs) |
| 256 | thresholds = check_trigger_thresholds(report) |
| 257 | |
| 258 | if report.sufficient: |
| 259 | print(f"✅ Baseline sufficient: {report.total_runs} runs (minimum: {args.min_runs})") |
| 260 | for stage in report.stages: |
| 261 | if stage.sample_count > 0: |
| 262 | print(f" {stage.stage}: p50={stage.p50_seconds}s p95={stage.p95_seconds}s") |
| 263 | else: |
| 264 | print( |
| 265 | f"❌ Baseline insufficient: {report.total_runs} runs " |
| 266 | f"(minimum: {args.min_runs} required)" |
| 267 | ) |
| 268 | return 1 |
| 269 | |
| 270 | # Check triggers |
| 271 | rss_triggered = thresholds["rss_matrix_triggers"]["triggered"] |
| 272 | print(f"\n RSS matrix trigger: {'🔴 TRIGGERED' if rss_triggered else '🟢 not triggered'}") |
| 273 | print(" GitHub shard trigger: 🟢 requires experiment comparison") |
| 274 | return 0 |
| 275 | |
| 276 | else: |
| 277 | parser.print_help() |
| 278 | return 1 |
| 279 | |
| 280 | |
| 281 | if __name__ == "__main__": |
| 282 | sys.exit(main()) |