| 1 | #!/usr/bin/env python3 |
| 2 | """Validate analysis predictions against later raw-star outcomes. |
| 3 | |
| 4 | Prediction registry format for future weekly summaries: |
| 5 | |
| 6 | ```yaml |
| 7 | predictions: |
| 8 | - repo: owner/repo |
| 9 | claim_type: signal |
| 10 | direction: up |
| 11 | confidence: 0.72 |
| 12 | ``` |
| 13 | |
| 14 | Required fields: |
| 15 | - `repo`: GitHub repo in `owner/name` form. |
| 16 | - `claim_type`: one of `signal`, `noise`, or `gap`. |
| 17 | - `direction`: one of `up`, `flat`, or `down`. |
| 18 | - `confidence`: float from 0.0 to 1.0. |
| 19 | |
| 20 | The validator will use frontmatter predictions when present. For legacy summaries |
| 21 | without a registry, it infers repo-level calls from Signal/Noise/Gaps prose. |
| 22 | It writes: |
| 23 | - `.squad/reskill/scorecards/YYYY-WNN.md` for editorial review |
| 24 | - `data/metrics/scorecards/YYYY-WNN-scorecard.json` for reskill ingestion |
| 25 | """ |
| 26 | |
| 27 | from __future__ import annotations |
| 28 | |
| 29 | import argparse |
| 30 | import json |
| 31 | import re |
| 32 | import sys |
| 33 | from dataclasses import asdict, dataclass |
| 34 | from datetime import UTC, datetime, timedelta |
| 35 | from pathlib import Path |
| 36 | from typing import Any |
| 37 | |
| 38 | ROOT = Path(__file__).resolve().parent.parent |
| 39 | if str(ROOT) not in sys.path: |
| 40 | sys.path.insert(0, str(ROOT)) |
| 41 | |
| 42 | from scripts import analysis_gate, track_quality # noqa: E402 |
| 43 | |
| 44 | DEFAULT_ANALYZED_DIR = ROOT / "data" / "analyzed" |
| 45 | DEFAULT_RAW_DIR = ROOT / "data" / "raw" |
| 46 | DEFAULT_METRICS_DIR = ROOT / "data" / "metrics" |
| 47 | DEFAULT_SCORECARD_DIR = ROOT / ".squad" / "reskill" / "scorecards" |
| 48 | DEFAULT_SNAPSHOTS_DIR = ROOT / "data" / "snapshots" |
| 49 | |
| 50 | REPO_LINK_PATTERN = re.compile( |
| 51 | r"\[(?P<repo>[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)\]\(https://github\.com/[^)]+\)" |
| 52 | ) |
| 53 | WEEK_PATTERN = re.compile(r"^(\d{4}-W\d{2})$") |
| 54 | RAW_WEEK_PATTERN = re.compile(r"^(\d{4}-W\d{2})\.json$") |
| 55 | SNAPSHOT_WEEK_PATTERN = re.compile(r"^(\d{4}-W\d{2})-stars\.json$") |
| 56 | HEADING_PATTERN = re.compile(r"(?m)^(#{2,3})\s+(.+?)\s*$") |
| 57 | SIGNAL_HINTS = ("durable signal", "strongest signal", "credible signal", "signal this week") |
| 58 | NOISE_HINTS = ( |
| 59 | "noise this week", |
| 60 | "the noise", |
| 61 | "coordination", |
| 62 | "spam cluster", |
| 63 | "manipulation campaign", |
| 64 | ) |
| 65 | TOP_REPO_PATTERN = re.compile(r"^[^/\s]+/[^/\s]+$") |
| 66 | |
| 67 | |
| 68 | @dataclass(frozen=True) |
| 69 | class Prediction: |
| 70 | week: str |
| 71 | repo: str |
| 72 | claim: str |
| 73 | direction: str |
| 74 | confidence: float |
| 75 | source: str |
| 76 | source_path: str |
| 77 | |
| 78 | |
| 79 | @dataclass(frozen=True) |
| 80 | class ValidationResult: |
| 81 | week: str |
| 82 | repo: str |
| 83 | claim: str |
| 84 | direction: str |
| 85 | confidence: float |
| 86 | source: str |
| 87 | source_path: str |
| 88 | baseline_week: str |
| 89 | observed_week: str | None |
| 90 | weeks_observed: int |
| 91 | baseline_stars: int | None |
| 92 | observed_stars: int | None |
| 93 | delta_stars: int | None |
| 94 | delta_pct: float | None |
| 95 | score: float | None |
| 96 | verdict: str |
| 97 | note: str |
| 98 | |
| 99 | |
| 100 | @dataclass(frozen=True) |
| 101 | class ScorecardSummary: |
| 102 | week: str |
| 103 | date: str |
| 104 | total_predictions: int |
| 105 | validated: int |
| 106 | correct: int |
| 107 | incorrect: int |
| 108 | accuracy: float |
| 109 | by_type: dict[str, dict[str, float | int]] |
| 110 | by_direction: dict[str, dict[str, float | int]] |
| 111 | quality_trend: dict[str, Any] |
| 112 | details: list[dict[str, Any]] |
| 113 | insufficient_evidence: list[dict[str, Any]] |
| 114 | |
| 115 | |
| 116 | class ValidationError(ValueError): |
| 117 | """Raised when a prediction registry entry is malformed.""" |
| 118 | |
| 119 | |
| 120 | def parse_args(argv: list[str] | None = None) -> argparse.Namespace: |
| 121 | parser = argparse.ArgumentParser( |
| 122 | description="Validate weekly Signal/Noise/Gaps calls against later raw star data." |
| 123 | ) |
| 124 | parser.add_argument( |
| 125 | "--analyzed-dir", |
| 126 | type=Path, |
| 127 | default=DEFAULT_ANALYZED_DIR, |
| 128 | help="Directory containing analyzed summaries.", |
| 129 | ) |
| 130 | parser.add_argument( |
| 131 | "--raw-dir", |
| 132 | type=Path, |
| 133 | default=DEFAULT_RAW_DIR, |
| 134 | help="Directory containing raw weekly JSON payloads.", |
| 135 | ) |
| 136 | parser.add_argument( |
| 137 | "--snapshots-dir", |
| 138 | type=Path, |
| 139 | default=DEFAULT_SNAPSHOTS_DIR, |
| 140 | help="Optional legacy snapshots directory; used when raw weekly JSON is unavailable.", |
| 141 | ) |
| 142 | parser.add_argument( |
| 143 | "--metrics-dir", |
| 144 | type=Path, |
| 145 | default=DEFAULT_METRICS_DIR, |
| 146 | help="Directory for machine-readable scorecards.", |
| 147 | ) |
| 148 | parser.add_argument( |
| 149 | "--scorecard-dir", |
| 150 | "--scorecards-dir", |
| 151 | dest="scorecard_dir", |
| 152 | type=Path, |
| 153 | default=DEFAULT_SCORECARD_DIR, |
| 154 | help="Directory for markdown scorecards.", |
| 155 | ) |
| 156 | parser.add_argument( |
| 157 | "--weeks-ahead", |
| 158 | type=int, |
| 159 | default=4, |
| 160 | help="Maximum lookahead window in ISO weeks (default: 4).", |
| 161 | ) |
| 162 | parser.add_argument( |
| 163 | "--report-week", help="Override the scorecard week slug. Defaults to the current ISO week." |
| 164 | ) |
| 165 | parser.add_argument( |
| 166 | "--current-datetime", help="Optional ISO timestamp used to derive the default report week." |
| 167 | ) |
| 168 | return parser.parse_args(argv) |
| 169 | |
| 170 | |
| 171 | def iso_week_to_date(week_str: str) -> datetime: |
| 172 | year, week_num = week_str.split("-W") |
| 173 | return datetime.strptime(f"{year}-W{int(week_num):02d}-1", "%G-W%V-%u").replace(tzinfo=UTC) |
| 174 | |
| 175 | |
| 176 | def current_iso_week(now: datetime | None = None) -> str: |
| 177 | current = now or datetime.now(tz=UTC) |
| 178 | year, week, _ = current.isocalendar() |
| 179 | return f"{year}-W{week:02d}" |
| 180 | |
| 181 | |
| 182 | def week_offset(week_str: str, offset: int) -> str: |
| 183 | shifted = iso_week_to_date(week_str) + timedelta(weeks=offset) |
| 184 | year, week, _ = shifted.isocalendar() |
| 185 | return f"{year}-W{week:02d}" |
| 186 | |
| 187 | |
| 188 | def week_distance(start_week: str, end_week: str) -> int: |
| 189 | return int((iso_week_to_date(end_week) - iso_week_to_date(start_week)).days / 7) |
| 190 | |
| 191 | |
| 192 | def load_json_file(path: Path) -> dict[str, Any] | None: |
| 193 | if not path.exists(): |
| 194 | return None |
| 195 | try: |
| 196 | return json.loads(path.read_text(encoding="utf-8")) |
| 197 | except (json.JSONDecodeError, OSError): |
| 198 | return None |
| 199 | |
| 200 | |
| 201 | def load_raw_week(raw_directory: Path, week_str: str) -> dict[str, Any] | None: |
| 202 | return load_json_file(raw_directory / f"{week_str}.json") |
| 203 | |
| 204 | |
| 205 | def load_snapshot_week(snapshot_directory: Path | None, week_str: str) -> dict[str, Any] | None: |
| 206 | if snapshot_directory is None: |
| 207 | return None |
| 208 | payload = load_json_file(snapshot_directory / f"{week_str}-stars.json") |
| 209 | if not payload: |
| 210 | return None |
| 211 | stars = payload.get("stars") |
| 212 | if not isinstance(stars, dict): |
| 213 | return None |
| 214 | repos = [{"full_name": repo, "stars": value} for repo, value in stars.items()] |
| 215 | return {"new_repos": repos, "trending_repos": []} |
| 216 | |
| 217 | |
| 218 | def list_available_raw_weeks( |
| 219 | raw_directory: Path, snapshot_directory: Path | None = None |
| 220 | ) -> list[str]: |
| 221 | weeks: set[str] = set() |
| 222 | if raw_directory.exists(): |
| 223 | for path in raw_directory.glob("*.json"): |
| 224 | match = RAW_WEEK_PATTERN.match(path.name) |
| 225 | if match: |
| 226 | weeks.add(match.group(1)) |
| 227 | if snapshot_directory and snapshot_directory.exists(): |
| 228 | for path in snapshot_directory.glob("*.json"): |
| 229 | match = SNAPSHOT_WEEK_PATTERN.match(path.name) |
| 230 | if match: |
| 231 | weeks.add(match.group(1)) |
| 232 | return sorted(weeks) |
| 233 | |
| 234 | |
| 235 | def build_repo_stars(raw_data: dict[str, Any]) -> dict[str, int]: |
| 236 | stars: dict[str, int] = {} |
| 237 | for section in ("new_repos", "trending_repos"): |
| 238 | for repo in raw_data.get(section, []): |
| 239 | full_name = repo.get("full_name") |
| 240 | if isinstance(full_name, str) and full_name: |
| 241 | stars[full_name] = int(repo.get("stars", 0) or 0) |
| 242 | return stars |
| 243 | |
| 244 | |
| 245 | def build_repo_set(raw_data: dict[str, Any]) -> set[str]: |
| 246 | return set(build_repo_stars(raw_data)) |
| 247 | |
| 248 | |
| 249 | def extract_repo_links(text: str) -> list[str]: |
| 250 | seen: set[str] = set() |
| 251 | repos: list[str] = [] |
| 252 | for match in REPO_LINK_PATTERN.finditer(text): |
| 253 | repo = match.group("repo") |
| 254 | if repo not in seen: |
| 255 | repos.append(repo) |
| 256 | seen.add(repo) |
| 257 | return repos |
| 258 | |
| 259 | |
| 260 | def split_markdown_sections(body: str) -> list[tuple[str, str]]: |
| 261 | matches = list(HEADING_PATTERN.finditer(body)) |
| 262 | sections: list[tuple[str, str]] = [] |
| 263 | for index, match in enumerate(matches): |
| 264 | start = match.end() |
| 265 | end = matches[index + 1].start() if index + 1 < len(matches) else len(body) |
| 266 | title = match.group(2).strip() |
| 267 | content = body[start:end].strip() |
| 268 | sections.append((title, content)) |
| 269 | return sections |
| 270 | |
| 271 | |
| 272 | def split_paragraphs(text: str) -> list[str]: |
| 273 | return [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()] |
| 274 | |
| 275 | |
| 276 | def normalize_direction(value: str) -> str: |
| 277 | direction = value.strip().lower() |
| 278 | if direction not in {"up", "flat", "down"}: |
| 279 | raise ValidationError(f"Unsupported prediction direction: {value!r}") |
| 280 | return direction |
| 281 | |
| 282 | |
| 283 | def normalize_confidence(value: Any) -> float: |
| 284 | if isinstance(value, bool): |
| 285 | raise ValidationError("confidence must be numeric, not boolean") |
| 286 | if not isinstance(value, (int, float)): |
| 287 | raise ValidationError("confidence must be numeric") |
| 288 | confidence = float(value) |
| 289 | if confidence < 0.0 or confidence > 1.0: |
| 290 | raise ValidationError("confidence must be between 0.0 and 1.0") |
| 291 | return round(confidence, 2) |
| 292 | |
| 293 | |
| 294 | def normalize_claim_type(value: str) -> str: |
| 295 | claim_type = value.strip().lower() |
| 296 | if claim_type not in {"signal", "noise", "gap"}: |
| 297 | raise ValidationError(f"Unsupported prediction claim_type: {value!r}") |
| 298 | return claim_type |
| 299 | |
| 300 | |
| 301 | def normalize_frontmatter_predictions( |
| 302 | frontmatter: dict[str, Any], week: str, source_path: str |
| 303 | ) -> list[Prediction]: |
| 304 | raw_predictions = frontmatter.get("predictions") |
| 305 | if raw_predictions is None: |
| 306 | return [] |
| 307 | if not isinstance(raw_predictions, list): |
| 308 | raise ValidationError("predictions frontmatter must be a list") |
| 309 | |
| 310 | predictions: list[Prediction] = [] |
| 311 | for entry in raw_predictions: |
| 312 | if not isinstance(entry, dict): |
| 313 | raise ValidationError("each predictions entry must be a mapping") |
| 314 | repo = entry.get("repo") |
| 315 | if not isinstance(repo, str) or not TOP_REPO_PATTERN.fullmatch(repo.strip()): |
| 316 | raise ValidationError("predictions repo must use owner/repo format") |
| 317 | claim = normalize_claim_type(str(entry.get("claim_type", ""))) |
| 318 | direction = normalize_direction(str(entry.get("direction", ""))) |
| 319 | confidence = normalize_confidence(entry.get("confidence")) |
| 320 | predictions.append( |
| 321 | Prediction( |
| 322 | week=week, |
| 323 | repo=repo.strip(), |
| 324 | claim=claim, |
| 325 | direction=direction, |
| 326 | confidence=confidence, |
| 327 | source="frontmatter", |
| 328 | source_path=source_path, |
| 329 | ) |
| 330 | ) |
| 331 | return predictions |
| 332 | |
| 333 | |
| 334 | def build_inferred_prediction(week: str, repo: str, claim: str, source_path: str) -> Prediction: |
| 335 | direction = {"signal": "up", "noise": "flat", "gap": "up"}[claim] |
| 336 | confidence = {"signal": 0.65, "noise": 0.7, "gap": 0.55}[claim] |
| 337 | return Prediction( |
| 338 | week=week, |
| 339 | repo=repo, |
| 340 | claim=claim, |
| 341 | direction=direction, |
| 342 | confidence=confidence, |
| 343 | source="inferred", |
| 344 | source_path=source_path, |
| 345 | ) |
| 346 | |
| 347 | |
| 348 | def infer_predictions_from_body(body: str, week: str, source_path: str) -> list[Prediction]: |
| 349 | predictions: list[Prediction] = [] |
| 350 | seen: set[tuple[str, str]] = set() |
| 351 | |
| 352 | def add_prediction(claim: str, repo: str) -> None: |
| 353 | key = (claim, repo) |
| 354 | if key in seen: |
| 355 | return |
| 356 | seen.add(key) |
| 357 | predictions.append(build_inferred_prediction(week, repo, claim, source_path)) |
| 358 | |
| 359 | for title, content in split_markdown_sections(body): |
| 360 | lower = title.lower() |
| 361 | if lower == "signal": |
| 362 | for repo in extract_repo_links(content): |
| 363 | add_prediction("signal", repo) |
| 364 | elif lower == "noise": |
| 365 | for repo in extract_repo_links(content): |
| 366 | add_prediction("noise", repo) |
| 367 | elif lower in {"gaps", "blind spots"} or "what's missing" in lower: |
| 368 | for repo in extract_repo_links(content): |
| 369 | add_prediction("gap", repo) |
| 370 | elif lower == "signal & noise": |
| 371 | for paragraph in split_paragraphs(content): |
| 372 | para_lower = paragraph.lower() |
| 373 | repos = extract_repo_links(paragraph) |
| 374 | if not repos: |
| 375 | continue |
| 376 | if any(hint in para_lower for hint in NOISE_HINTS): |
| 377 | for repo in repos: |
| 378 | add_prediction("noise", repo) |
| 379 | elif any(hint in para_lower for hint in SIGNAL_HINTS) or "signal" in para_lower: |
| 380 | for repo in repos: |
| 381 | add_prediction("signal", repo) |
| 382 | |
| 383 | return predictions |
| 384 | |
| 385 | |
| 386 | def load_summary_predictions(summary_path: Path) -> list[Prediction]: |
| 387 | text = summary_path.read_text(encoding="utf-8") |
| 388 | frontmatter, body = analysis_gate.extract_frontmatter(text) |
| 389 | week = frontmatter.get("week") |
| 390 | if not isinstance(week, str) or not WEEK_PATTERN.fullmatch(week): |
| 391 | return [] |
| 392 | |
| 393 | source_path = ( |
| 394 | str(summary_path.relative_to(ROOT)) |
| 395 | if summary_path.is_relative_to(ROOT) |
| 396 | else str(summary_path) |
| 397 | ) |
| 398 | frontmatter_predictions = normalize_frontmatter_predictions(frontmatter, week, source_path) |
| 399 | if frontmatter_predictions: |
| 400 | return frontmatter_predictions |
| 401 | return infer_predictions_from_body(body, week, source_path) |
| 402 | |
| 403 | |
| 404 | def expected_growth(direction: str, confidence: float, weeks_observed: int) -> float: |
| 405 | scale = max(weeks_observed, 1) / 4.0 |
| 406 | if direction == "up": |
| 407 | return max(0.03, (0.12 + (confidence * 0.28)) * scale) |
| 408 | if direction == "flat": |
| 409 | return max(0.02, (0.20 - (confidence * 0.12)) * scale) |
| 410 | return max(0.01, (0.05 - (confidence * 0.03)) * scale) |
| 411 | |
| 412 | |
| 413 | def locate_observed_week( |
| 414 | raw_directory: Path, |
| 415 | prediction_week: str, |
| 416 | weeks_ahead: int, |
| 417 | snapshot_directory: Path | None = None, |
| 418 | ) -> str | None: |
| 419 | available = list_available_raw_weeks(raw_directory, snapshot_directory) |
| 420 | candidates = [ |
| 421 | week |
| 422 | for week in available |
| 423 | if week_distance(prediction_week, week) > 0 |
| 424 | and week_distance(prediction_week, week) <= weeks_ahead |
| 425 | ] |
| 426 | return candidates[-1] if candidates else None |
| 427 | |
| 428 | |
| 429 | def evaluate_prediction( |
| 430 | prediction: Prediction, |
| 431 | raw_directory: Path, |
| 432 | weeks_ahead: int, |
| 433 | snapshot_directory: Path | None = None, |
| 434 | ) -> ValidationResult: |
| 435 | baseline_raw = load_raw_week(raw_directory, prediction.week) or load_snapshot_week( |
| 436 | snapshot_directory, prediction.week |
| 437 | ) |
| 438 | if baseline_raw is None: |
| 439 | return ValidationResult( |
| 440 | week=prediction.week, |
| 441 | repo=prediction.repo, |
| 442 | claim=prediction.claim, |
| 443 | direction=prediction.direction, |
| 444 | confidence=prediction.confidence, |
| 445 | source=prediction.source, |
| 446 | source_path=prediction.source_path, |
| 447 | baseline_week=prediction.week, |
| 448 | observed_week=None, |
| 449 | weeks_observed=0, |
| 450 | baseline_stars=None, |
| 451 | observed_stars=None, |
| 452 | delta_stars=None, |
| 453 | delta_pct=None, |
| 454 | score=None, |
| 455 | verdict="insufficient_evidence", |
| 456 | note="No raw payload exists for the prediction week.", |
| 457 | ) |
| 458 | |
| 459 | baseline_stars_map = build_repo_stars(baseline_raw) |
| 460 | if prediction.repo not in baseline_stars_map: |
| 461 | return ValidationResult( |
| 462 | week=prediction.week, |
| 463 | repo=prediction.repo, |
| 464 | claim=prediction.claim, |
| 465 | direction=prediction.direction, |
| 466 | confidence=prediction.confidence, |
| 467 | source=prediction.source, |
| 468 | source_path=prediction.source_path, |
| 469 | baseline_week=prediction.week, |
| 470 | observed_week=None, |
| 471 | weeks_observed=0, |
| 472 | baseline_stars=None, |
| 473 | observed_stars=None, |
| 474 | delta_stars=None, |
| 475 | delta_pct=None, |
| 476 | score=None, |
| 477 | verdict="insufficient_evidence", |
| 478 | note="Repo was not present in the prediction-week crawl, so no baseline comparison is possible.", |
| 479 | ) |
| 480 | |
| 481 | baseline_stars = baseline_stars_map[prediction.repo] |
| 482 | observed_week = locate_observed_week( |
| 483 | raw_directory, prediction.week, weeks_ahead, snapshot_directory |
| 484 | ) |
| 485 | if observed_week is None: |
| 486 | return ValidationResult( |
| 487 | week=prediction.week, |
| 488 | repo=prediction.repo, |
| 489 | claim=prediction.claim, |
| 490 | direction=prediction.direction, |
| 491 | confidence=prediction.confidence, |
| 492 | source=prediction.source, |
| 493 | source_path=prediction.source_path, |
| 494 | baseline_week=prediction.week, |
| 495 | observed_week=None, |
| 496 | weeks_observed=0, |
| 497 | baseline_stars=baseline_stars, |
| 498 | observed_stars=None, |
| 499 | delta_stars=None, |
| 500 | delta_pct=None, |
| 501 | score=None, |
| 502 | verdict="insufficient_evidence", |
| 503 | note="No later raw week is available inside the validation window.", |
| 504 | ) |
| 505 | |
| 506 | observed_raw = load_raw_week(raw_directory, observed_week) or load_snapshot_week( |
| 507 | snapshot_directory, observed_week |
| 508 | ) |
| 509 | if observed_raw is None: |
| 510 | return ValidationResult( |
| 511 | week=prediction.week, |
| 512 | repo=prediction.repo, |
| 513 | claim=prediction.claim, |
| 514 | direction=prediction.direction, |
| 515 | confidence=prediction.confidence, |
| 516 | source=prediction.source, |
| 517 | source_path=prediction.source_path, |
| 518 | baseline_week=prediction.week, |
| 519 | observed_week=observed_week, |
| 520 | weeks_observed=week_distance(prediction.week, observed_week), |
| 521 | baseline_stars=baseline_stars, |
| 522 | observed_stars=None, |
| 523 | delta_stars=None, |
| 524 | delta_pct=None, |
| 525 | score=None, |
| 526 | verdict="insufficient_evidence", |
| 527 | note="Later raw payload could not be parsed.", |
| 528 | ) |
| 529 | |
| 530 | observed_stars_map = build_repo_stars(observed_raw) |
| 531 | observed_set = set(observed_stars_map) |
| 532 | weeks_observed = max(week_distance(prediction.week, observed_week), 1) |
| 533 | |
| 534 | if prediction.repo not in observed_set: |
| 535 | return ValidationResult( |
| 536 | week=prediction.week, |
| 537 | repo=prediction.repo, |
| 538 | claim=prediction.claim, |
| 539 | direction=prediction.direction, |
| 540 | confidence=prediction.confidence, |
| 541 | source=prediction.source, |
| 542 | source_path=prediction.source_path, |
| 543 | baseline_week=prediction.week, |
| 544 | observed_week=observed_week, |
| 545 | weeks_observed=weeks_observed, |
| 546 | baseline_stars=baseline_stars, |
| 547 | observed_stars=None, |
| 548 | delta_stars=None, |
| 549 | delta_pct=None, |
| 550 | score=None, |
| 551 | verdict="insufficient_evidence", |
| 552 | note="Repo was not present in the later crawl payload, so the observation window is inconclusive.", |
| 553 | ) |
| 554 | |
| 555 | observed_stars = observed_stars_map[prediction.repo] |
| 556 | delta_stars = observed_stars - baseline_stars |
| 557 | delta_pct = ( |
| 558 | (delta_stars / baseline_stars) |
| 559 | if baseline_stars > 0 |
| 560 | else (1.0 if observed_stars > 0 else 0.0) |
| 561 | ) |
| 562 | threshold = expected_growth(prediction.direction, prediction.confidence, weeks_observed) |
| 563 | |
| 564 | if prediction.direction == "up": |
| 565 | score = min(1.0, max(0.0, delta_pct / threshold)) |
| 566 | verdict = "correct" if score >= 0.6 else "incorrect" |
| 567 | note = f"Expected at least {threshold:.1%} growth over the observed window; saw {delta_pct:.1%}." |
| 568 | else: |
| 569 | overshoot = max(0.0, delta_pct - threshold) |
| 570 | denominator = max(0.05, threshold) |
| 571 | score = max(0.0, 1.0 - (overshoot / denominator)) |
| 572 | verdict = "correct" if score >= 0.6 else "incorrect" |
| 573 | comparator = "flat" if prediction.direction == "flat" else "down" |
| 574 | note = f"Expected {comparator} performance with at most {threshold:.1%} growth; saw {delta_pct:.1%}." |
| 575 | |
| 576 | return ValidationResult( |
| 577 | week=prediction.week, |
| 578 | repo=prediction.repo, |
| 579 | claim=prediction.claim, |
| 580 | direction=prediction.direction, |
| 581 | confidence=prediction.confidence, |
| 582 | source=prediction.source, |
| 583 | source_path=prediction.source_path, |
| 584 | baseline_week=prediction.week, |
| 585 | observed_week=observed_week, |
| 586 | weeks_observed=weeks_observed, |
| 587 | baseline_stars=baseline_stars, |
| 588 | observed_stars=observed_stars, |
| 589 | delta_stars=delta_stars, |
| 590 | delta_pct=round(delta_pct, 4), |
| 591 | score=round(score, 4), |
| 592 | verdict=verdict, |
| 593 | note=note, |
| 594 | ) |
| 595 | |
| 596 | |
| 597 | def summarize_bucket( |
| 598 | results: list[ValidationResult], key: str |
| 599 | ) -> dict[str, dict[str, float | int]]: |
| 600 | summary: dict[str, dict[str, float | int]] = {} |
| 601 | for result in results: |
| 602 | bucket = getattr(result, key) |
| 603 | entry = summary.setdefault( |
| 604 | bucket, {"total": 0, "correct": 0, "incorrect": 0, "accuracy": 0.0} |
| 605 | ) |
| 606 | entry["total"] += 1 |
| 607 | if result.verdict == "correct": |
| 608 | entry["correct"] += 1 |
| 609 | elif result.verdict == "incorrect": |
| 610 | entry["incorrect"] += 1 |
| 611 | for entry in summary.values(): |
| 612 | total = int(entry["total"]) |
| 613 | correct = int(entry["correct"]) |
| 614 | entry["accuracy"] = round((correct / total), 4) if total else 0.0 |
| 615 | return summary |
| 616 | |
| 617 | |
| 618 | def quality_trend_summary(analyzed_dir: Path) -> dict[str, Any]: |
| 619 | entries = track_quality.load_quality_entries(analyzed_dir) |
| 620 | if not entries: |
| 621 | return { |
| 622 | "count": 0, |
| 623 | "average": 0.0, |
| 624 | "trend": "insufficient history", |
| 625 | "latest_week": None, |
| 626 | "latest_score": None, |
| 627 | "best_week": None, |
| 628 | "best_score": None, |
| 629 | "lowest_week": None, |
| 630 | "lowest_score": None, |
| 631 | } |
| 632 | average = round(sum(entry.score for entry in entries) / len(entries), 1) |
| 633 | best = max(entries, key=lambda entry: entry.score) |
| 634 | worst = min(entries, key=lambda entry: entry.score) |
| 635 | latest = entries[-1] |
| 636 | return { |
| 637 | "count": len(entries), |
| 638 | "average": average, |
| 639 | "trend": track_quality.classify_trend(entries), |
| 640 | "latest_week": latest.week, |
| 641 | "latest_score": latest.score, |
| 642 | "best_week": best.week, |
| 643 | "best_score": best.score, |
| 644 | "lowest_week": worst.week, |
| 645 | "lowest_score": worst.score, |
| 646 | } |
| 647 | |
| 648 | |
| 649 | def build_scorecard( |
| 650 | results: list[ValidationResult], analyzed_dir: Path, report_week: str |
| 651 | ) -> ScorecardSummary: |
| 652 | validated = [result for result in results if result.verdict in {"correct", "incorrect"}] |
| 653 | insufficient = [result for result in results if result.verdict == "insufficient_evidence"] |
| 654 | correct = sum(1 for result in validated if result.verdict == "correct") |
| 655 | incorrect = sum(1 for result in validated if result.verdict == "incorrect") |
| 656 | accuracy = round((correct / len(validated)), 4) if validated else 0.0 |
| 657 | return ScorecardSummary( |
| 658 | week=report_week, |
| 659 | date=datetime.now(tz=UTC).date().isoformat(), |
| 660 | total_predictions=len(results), |
| 661 | validated=len(validated), |
| 662 | correct=correct, |
| 663 | incorrect=incorrect, |
| 664 | accuracy=accuracy, |
| 665 | by_type=summarize_bucket(validated, "claim"), |
| 666 | by_direction=summarize_bucket(validated, "direction"), |
| 667 | quality_trend=quality_trend_summary(analyzed_dir), |
| 668 | details=[asdict(result) for result in validated], |
| 669 | insufficient_evidence=[asdict(result) for result in insufficient], |
| 670 | ) |
| 671 | |
| 672 | |
| 673 | def render_percentage(value: float) -> str: |
| 674 | return f"{int(round(value * 100))}%" |
| 675 | |
| 676 | |
| 677 | def render_quality_block(quality: dict[str, Any]) -> list[str]: |
| 678 | if not quality["count"]: |
| 679 | return ["No `quality_score` history is available yet."] |
| 680 | return [ |
| 681 | f"- Summaries tracked: {quality['count']}", |
| 682 | f"- Average quality score: {quality['average']}", |
| 683 | f"- Trend: {quality['trend']}", |
| 684 | f"- Latest week: {quality['latest_week']} ({quality['latest_score']})", |
| 685 | f"- Best week: {quality['best_week']} ({quality['best_score']})", |
| 686 | f"- Lowest week: {quality['lowest_week']} ({quality['lowest_score']})", |
| 687 | ] |
| 688 | |
| 689 | |
| 690 | def render_accuracy_table(summary: dict[str, dict[str, float | int]], label: str) -> list[str]: |
| 691 | if not summary: |
| 692 | return [f"No validated {label.lower()} calls yet."] |
| 693 | lines = [ |
| 694 | f"| {label} | Correct | Incorrect | Total | Accuracy |", |
| 695 | "| --- | ---: | ---: | ---: | ---: |", |
| 696 | ] |
| 697 | for bucket, stats in sorted(summary.items()): |
| 698 | lines.append( |
| 699 | f"| {bucket} | {int(stats['correct'])} | {int(stats['incorrect'])} | {int(stats['total'])} | {render_percentage(float(stats['accuracy']))} |" |
| 700 | ) |
| 701 | return lines |
| 702 | |
| 703 | |
| 704 | def render_details_table(results: list[dict[str, Any]]) -> list[str]: |
| 705 | if not results: |
| 706 | return ["No predictions have enough later data to score yet."] |
| 707 | lines = [ |
| 708 | "| Week | Repo | Claim | Direction | Conf. | Baseline | Observed | Δ Stars | Verdict | Note |", |
| 709 | "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |", |
| 710 | ] |
| 711 | for result in results: |
| 712 | note = str(result["note"]).replace("|", "\\|") |
| 713 | observed_week = result.get("observed_week") or "—" |
| 714 | observed_stars = result.get("observed_stars") |
| 715 | lines.append( |
| 716 | f"| {result['week']} → {observed_week} | {result['repo']} | {result['claim']} | {result['direction']} | {float(result['confidence']):.2f} | {result.get('baseline_stars', '—')} | {observed_stars if observed_stars is not None else '—'} | {result.get('delta_stars', '—')} | {result['verdict']} | {note} |" |
| 717 | ) |
| 718 | return lines |
| 719 | |
| 720 | |
| 721 | def render_markdown_scorecard(scorecard: ScorecardSummary, weeks_ahead: int) -> str: |
| 722 | lines = [ |
| 723 | f"# Prediction Scorecard: {scorecard.week}", |
| 724 | "", |
| 725 | f"- Date: {scorecard.date}", |
| 726 | f"- Validation window: up to {weeks_ahead} weeks later, using the furthest raw week available", |
| 727 | f"- Predictions found: {scorecard.total_predictions}", |
| 728 | f"- Predictions validated: {scorecard.validated}", |
| 729 | f"- Overall accuracy: {render_percentage(scorecard.accuracy)} ({scorecard.correct}/{scorecard.validated if scorecard.validated else 0} correct)", |
| 730 | "", |
| 731 | "## Prediction Registry Format", |
| 732 | "", |
| 733 | "Use `predictions: [{repo, claim_type, direction, confidence}]` in analysis frontmatter. `claim_type` must be `signal`, `noise`, or `gap`; `direction` must be `up`, `flat`, or `down`. When the registry is absent, the validator still infers Signal/Noise/Gaps from article sections.", |
| 734 | "", |
| 735 | "## Quality Trend", |
| 736 | "", |
| 737 | *render_quality_block(scorecard.quality_trend), |
| 738 | "", |
| 739 | "## Accuracy by Claim", |
| 740 | "", |
| 741 | *render_accuracy_table(scorecard.by_type, "Claim"), |
| 742 | "", |
| 743 | "## Accuracy by Direction", |
| 744 | "", |
| 745 | *render_accuracy_table(scorecard.by_direction, "Direction"), |
| 746 | "", |
| 747 | "## Validated Calls", |
| 748 | "", |
| 749 | *render_details_table(scorecard.details), |
| 750 | "", |
| 751 | "## Insufficient Evidence", |
| 752 | "", |
| 753 | ] |
| 754 | if scorecard.insufficient_evidence: |
| 755 | lines.extend( |
| 756 | [ |
| 757 | f"- `{result['week']}` `{result['repo']}` ({result['claim']}/{result['direction']}) — {result['note']}" |
| 758 | for result in scorecard.insufficient_evidence |
| 759 | ] |
| 760 | ) |
| 761 | else: |
| 762 | lines.append("- None.") |
| 763 | lines.extend( |
| 764 | [ |
| 765 | "", |
| 766 | "## Editorial Readout", |
| 767 | "", |
| 768 | "Signal calls are judged by later star growth, noise calls by limited follow-on growth or disappearance, and gap calls by the traction of the related edge repos cited in the blind-spot narrative. Gap scores are therefore weaker proxies than signal/noise scores and should guide reskill discussion rather than act as hard truth.", |
| 769 | "", |
| 770 | ] |
| 771 | ) |
| 772 | return "\n".join(lines) |
| 773 | |
| 774 | |
| 775 | def save_json_scorecard(scorecard: ScorecardSummary, metrics_dir: Path) -> Path: |
| 776 | scorecards_dir = metrics_dir / "scorecards" |
| 777 | scorecards_dir.mkdir(parents=True, exist_ok=True) |
| 778 | path = scorecards_dir / f"{scorecard.week}-scorecard.json" |
| 779 | payload = asdict(scorecard) |
| 780 | payload["total_validated"] = scorecard.validated |
| 781 | path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") |
| 782 | return path |
| 783 | |
| 784 | |
| 785 | def save_markdown_scorecard(markdown: str, scorecard_dir: Path, report_week: str) -> Path: |
| 786 | scorecard_dir.mkdir(parents=True, exist_ok=True) |
| 787 | path = scorecard_dir / f"{report_week}.md" |
| 788 | path.write_text(markdown, encoding="utf-8") |
| 789 | return path |
| 790 | |
| 791 | |
| 792 | def run_validation( |
| 793 | analyzed_dir: Path = DEFAULT_ANALYZED_DIR, |
| 794 | raw_dir: Path = DEFAULT_RAW_DIR, |
| 795 | metrics_dir: Path = DEFAULT_METRICS_DIR, |
| 796 | scorecard_dir: Path = DEFAULT_SCORECARD_DIR, |
| 797 | weeks_ahead: int = 4, |
| 798 | report_week: str | None = None, |
| 799 | snapshot_dir: Path | None = DEFAULT_SNAPSHOTS_DIR, |
| 800 | ) -> ScorecardSummary: |
| 801 | predictions: list[Prediction] = [] |
| 802 | for summary_path in sorted(analyzed_dir.glob("*-summary.md")): |
| 803 | predictions.extend(load_summary_predictions(summary_path)) |
| 804 | |
| 805 | results = [ |
| 806 | evaluate_prediction(prediction, raw_dir, weeks_ahead, snapshot_dir) |
| 807 | for prediction in predictions |
| 808 | ] |
| 809 | summary = build_scorecard(results, analyzed_dir, report_week or current_iso_week()) |
| 810 | save_json_scorecard(summary, metrics_dir) |
| 811 | save_markdown_scorecard( |
| 812 | render_markdown_scorecard(summary, weeks_ahead), scorecard_dir, summary.week |
| 813 | ) |
| 814 | return summary |
| 815 | |
| 816 | |
| 817 | def main(argv: list[str] | None = None) -> int: |
| 818 | args = parse_args(argv) |
| 819 | now = ( |
| 820 | datetime.fromisoformat(args.current_datetime.replace("Z", "+00:00")) |
| 821 | if args.current_datetime |
| 822 | else None |
| 823 | ) |
| 824 | summary = run_validation( |
| 825 | analyzed_dir=args.analyzed_dir, |
| 826 | raw_dir=args.raw_dir, |
| 827 | metrics_dir=args.metrics_dir, |
| 828 | scorecard_dir=args.scorecard_dir, |
| 829 | weeks_ahead=args.weeks_ahead, |
| 830 | report_week=args.report_week or current_iso_week(now), |
| 831 | snapshot_dir=args.snapshots_dir, |
| 832 | ) |
| 833 | print( |
| 834 | f"Validated {summary.validated} of {summary.total_predictions} predictions for {summary.week}." |
| 835 | ) |
| 836 | print( |
| 837 | f"Accuracy: {render_percentage(summary.accuracy)} ({summary.correct}/{summary.validated if summary.validated else 0})." |
| 838 | ) |
| 839 | return 0 |
| 840 | |
| 841 | |
| 842 | if __name__ == "__main__": |
| 843 | raise SystemExit(main()) |