feat: per-topic learning state and seeded wisdom (#66) (#100)
- Add scripts/init_topic_learning.py for initializing per-topic learning dirs - Seed wisdom files for ai-ml and rust topics - Create .squad/topics/{topic_id}/ structure with wisdom.md, skills/, scorecards/ - Script supports --topic, --config, and --force flags - Idempotent: safe to run multiple times without data loss Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 19, 2026 at 15:55 UTC
c8838b97e09829fbdb9c8b6fc6d23f1f0708d08c
2 files changed
+746
scripts/hindsight_validation.py
new
+383
@@ -0,0 +1,383 @@
1
+#!/usr/bin/env python3
2
+"""Validate predictions from N weeks ago against actual outcomes.
3
+
4
+Reads predictions.jsonl, finds unvalidated predictions older than --weeks-ago,
5
+checks whether predicted outcomes occurred by comparing raw data from the
6
+prediction week against subsequent weeks, and writes a scorecard summary.
7
+
8
+Usage:
9
+ python scripts/hindsight_validation.py [--topic ai-ml] [--weeks-ago 4] [--data-dir data/]
10
+"""
11
+
12
+from __future__ import annotations
13
+
14
+import argparse
15
+import json
16
+from datetime import datetime, timedelta
17
+from pathlib import Path
18
+from typing import Any
19
+
20
+from scripts.topic_paths import metrics_dir, raw_dir
21
+
22
+
23
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
24
+ parser = argparse.ArgumentParser(
25
+ description="Validate predictions against actual outcomes."
26
+ )
27
+ parser.add_argument(
28
+ "--topic",
29
+ default=None,
30
+ help="Topic ID for path resolution. Defaults to general.",
31
+ )
32
+ parser.add_argument(
33
+ "--weeks-ago",
34
+ type=int,
35
+ default=4,
36
+ help="Minimum age in weeks for predictions to validate (default: 4).",
37
+ )
38
+ parser.add_argument(
39
+ "--data-dir",
40
+ default="data/",
41
+ help="Base data directory (default: data/).",
42
+ )
43
+ return parser.parse_args(argv)
44
+
45
+
46
+def iso_week_to_date(week_str: str) -> datetime:
47
+ """Convert YYYY-WNN to a datetime (Monday of that week)."""
48
+ year, week_num = week_str.split("-W")
49
+ return datetime.strptime(f"{year}-W{int(week_num):02d}-1", "%G-W%V-%u")
50
+
51
+
52
+def current_iso_week() -> str:
53
+ """Return the current ISO week as YYYY-WNN."""
54
+ now = datetime.now()
55
+ cal = now.isocalendar()
56
+ return f"{cal[0]}-W{cal[1]:02d}"
57
+
58
+
59
+def week_offset(week_str: str, offset: int) -> str:
60
+ """Return a week string offset by N weeks."""
61
+ dt = iso_week_to_date(week_str)
62
+ new_dt = dt + timedelta(weeks=offset)
63
+ cal = new_dt.isocalendar()
64
+ return f"{cal[0]}-W{cal[1]:02d}"
65
+
66
+
67
+def load_predictions(path: Path) -> list[dict[str, Any]]:
68
+ """Load predictions from a JSONL file."""
69
+ if not path.exists():
70
+ return []
71
+ predictions = []
72
+ with open(path, encoding="utf-8") as f:
73
+ for line in f:
74
+ line = line.strip()
75
+ if line:
76
+ predictions.append(json.loads(line))
77
+ return predictions
78
+
79
+
80
+def save_predictions(predictions: list[dict[str, Any]], path: Path) -> None:
81
+ """Write predictions back to a JSONL file."""
82
+ path.parent.mkdir(parents=True, exist_ok=True)
83
+ with open(path, "w", encoding="utf-8") as f:
84
+ for pred in predictions:
85
+ f.write(json.dumps(pred, ensure_ascii=False) + "\n")
86
+
87
+
88
+def load_raw_week(raw_directory: Path, week_str: str) -> dict[str, Any] | None:
89
+ """Load raw JSON for a given week. Returns None if missing."""
90
+ path = raw_directory / f"{week_str}.json"
91
+ if not path.exists():
92
+ return None
93
+ try:
94
+ with open(path, encoding="utf-8") as f:
95
+ return json.load(f)
96
+ except (json.JSONDecodeError, OSError):
97
+ return None
98
+
99
+
100
+def build_repo_stars(raw_data: dict[str, Any]) -> dict[str, int]:
101
+ """Extract repo -> stars mapping from raw data."""
102
+ stars: dict[str, int] = {}
103
+ for section in ("new_repos", "trending_repos"):
104
+ for repo in raw_data.get(section, []):
105
+ name = repo.get("full_name", "")
106
+ if name:
107
+ stars[name] = repo.get("stars", 0)
108
+ return stars
109
+
110
+
111
+def build_repo_set(raw_data: dict[str, Any]) -> set[str]:
112
+ """Extract the set of repo names present in raw data."""
113
+ repos: set[str] = set()
114
+ for section in ("new_repos", "trending_repos"):
115
+ for repo in raw_data.get(section, []):
116
+ name = repo.get("full_name", "")
117
+ if name:
118
+ repos.add(name)
119
+ return repos
120
+
121
+
122
+def find_subsequent_weeks(raw_directory: Path, start_week: str, count: int) -> list[str]:
123
+ """Find up to `count` weeks of raw data after start_week."""
124
+ weeks = []
125
+ for i in range(1, count + 1):
126
+ w = week_offset(start_week, i)
127
+ if (raw_directory / f"{w}.json").exists():
128
+ weeks.append(w)
129
+ return weeks
130
+
131
+
132
+def validate_rising_star(
133
+ repo: str,
134
+ prediction_stars: int,
135
+ raw_directory: Path,
136
+ prediction_week: str,
137
+ weeks_ahead: int,
138
+) -> bool | None:
139
+ """Validate rising_star: stars grew 20%+ in subsequent weeks."""
140
+ for i in range(weeks_ahead, 0, -1):
141
+ w = week_offset(prediction_week, i)
142
+ raw_data = load_raw_week(raw_directory, w)
143
+ if raw_data is None:
144
+ continue
145
+ current_stars = build_repo_stars(raw_data).get(repo)
146
+ if current_stars is not None:
147
+ if prediction_stars == 0:
148
+ return current_stars > 0
149
+ growth = (current_stars - prediction_stars) / prediction_stars
150
+ return growth >= 0.20
151
+ return None # no data to validate
152
+
153
+
154
+def validate_breakout_candidate(
155
+ repo: str,
156
+ raw_directory: Path,
157
+ prediction_week: str,
158
+ weeks_ahead: int,
159
+) -> bool | None:
160
+ """Validate breakout_candidate: appeared in trending in subsequent weeks."""
161
+ for i in range(1, weeks_ahead + 1):
162
+ w = week_offset(prediction_week, i)
163
+ raw_data = load_raw_week(raw_directory, w)
164
+ if raw_data is None:
165
+ continue
166
+ trending = {
167
+ r.get("full_name", "")
168
+ for r in raw_data.get("trending_repos", [])
169
+ }
170
+ if repo in trending:
171
+ return True
172
+ return False
173
+
174
+
175
+def validate_momentum_shift(
176
+ repo: str,
177
+ prediction_stars: int,
178
+ raw_directory: Path,
179
+ prediction_week: str,
180
+ weeks_ahead: int,
181
+) -> bool | None:
182
+ """Validate momentum_shift: trend continued in same direction."""
183
+ star_history = [prediction_stars]
184
+ for i in range(1, weeks_ahead + 1):
185
+ w = week_offset(prediction_week, i)
186
+ raw_data = load_raw_week(raw_directory, w)
187
+ if raw_data is None:
188
+ continue
189
+ s = build_repo_stars(raw_data).get(repo)
190
+ if s is not None:
191
+ star_history.append(s)
192
+
193
+ if len(star_history) < 2:
194
+ return None
195
+ # Check if direction is consistent (non-decreasing)
196
+ return star_history[-1] >= star_history[0]
197
+
198
+
199
+def validate_declining_signal(
200
+ repo: str,
201
+ raw_directory: Path,
202
+ prediction_week: str,
203
+ weeks_ahead: int,
204
+) -> bool | None:
205
+ """Validate declining_signal: repo disappeared from subsequent crawls."""
206
+ found_count = 0
207
+ checked_count = 0
208
+ for i in range(1, weeks_ahead + 1):
209
+ w = week_offset(prediction_week, i)
210
+ raw_data = load_raw_week(raw_directory, w)
211
+ if raw_data is None:
212
+ continue
213
+ checked_count += 1
214
+ if repo in build_repo_set(raw_data):
215
+ found_count += 1
216
+
217
+ if checked_count == 0:
218
+ return None
219
+ # Validated if repo is absent from majority of subsequent crawls
220
+ return found_count <= checked_count // 2
221
+
222
+
223
+def validate_prediction(
224
+ prediction: dict[str, Any],
225
+ raw_directory: Path,
226
+ weeks_ahead: int,
227
+) -> bool | None:
228
+ """Validate a single prediction. Returns True/False or None if insufficient data."""
229
+ repo = prediction.get("repo", "")
230
+ prediction_week = prediction.get("week", "")
231
+ pred_type = prediction.get("prediction", "")
232
+
233
+ if not repo or not prediction_week:
234
+ return None
235
+
236
+ # Load prediction week data to get baseline stars
237
+ pred_raw = load_raw_week(raw_directory, prediction_week)
238
+ prediction_stars = 0
239
+ if pred_raw:
240
+ prediction_stars = build_repo_stars(pred_raw).get(repo, 0)
241
+
242
+ if pred_type == "rising_star":
243
+ return validate_rising_star(
244
+ repo, prediction_stars, raw_directory, prediction_week, weeks_ahead
245
+ )
246
+ elif pred_type == "breakout_candidate":
247
+ return validate_breakout_candidate(
248
+ repo, raw_directory, prediction_week, weeks_ahead
249
+ )
250
+ elif pred_type == "momentum_shift":
251
+ return validate_momentum_shift(
252
+ repo, prediction_stars, raw_directory, prediction_week, weeks_ahead
253
+ )
254
+ elif pred_type in ("declining_signal", "emerging_topic"):
255
+ return validate_declining_signal(
256
+ repo, raw_directory, prediction_week, weeks_ahead
257
+ )
258
+ return None
259
+
260
+
261
+def is_old_enough(prediction_week: str, weeks_ago: int) -> bool:
262
+ """Check if a prediction is at least weeks_ago weeks old."""
263
+ try:
264
+ pred_date = iso_week_to_date(prediction_week)
265
+ cutoff = datetime.now() - timedelta(weeks=weeks_ago)
266
+ return pred_date <= cutoff
267
+ except (ValueError, AttributeError):
268
+ return False
269
+
270
+
271
+def generate_scorecard(predictions: list[dict[str, Any]]) -> dict[str, Any]:
272
+ """Generate a scorecard summary from validated predictions."""
273
+ validated = [p for p in predictions if p.get("validated") is not None]
274
+ correct = [p for p in validated if p.get("validated") is True]
275
+
276
+ by_type: dict[str, dict[str, int]] = {}
277
+ for p in validated:
278
+ t = p.get("prediction", "unknown")
279
+ if t not in by_type:
280
+ by_type[t] = {"total": 0, "correct": 0}
281
+ by_type[t]["total"] += 1
282
+ if p.get("validated") is True:
283
+ by_type[t]["correct"] += 1
284
+
285
+ total = len(validated)
286
+ accuracy = round(len(correct) / total, 4) if total > 0 else 0.0
287
+
288
+ return {
289
+ "total_validated": total,
290
+ "correct": len(correct),
291
+ "incorrect": total - len(correct),
292
+ "accuracy": accuracy,
293
+ "by_type": {
294
+ k: {
295
+ **v,
296
+ "accuracy": round(v["correct"] / v["total"], 4) if v["total"] > 0 else 0.0,
297
+ }
298
+ for k, v in by_type.items()
299
+ },
300
+ }
301
+
302
+
303
+def save_scorecard(scorecard: dict[str, Any], metrics_directory: Path) -> Path:
304
+ """Save scorecard to data/metrics/{topic}/scorecards/YYYY-WNN-scorecard.json."""
305
+ week = current_iso_week()
306
+ scorecards_dir = metrics_directory / "scorecards"
307
+ scorecards_dir.mkdir(parents=True, exist_ok=True)
308
+ path = scorecards_dir / f"{week}-scorecard.json"
309
+ with open(path, "w", encoding="utf-8") as f:
310
+ json.dump(scorecard, f, indent=2, ensure_ascii=False)
311
+ return path
312
+
313
+
314
+def run_validation(
315
+ topic_id: str | None = None,
316
+ weeks_ago: int = 4,
317
+ data_dir: str = "data/",
318
+) -> dict[str, Any]:
319
+ """Main validation logic. Returns the scorecard."""
320
+ import scripts.topic_paths as tp
321
+
322
+ # Override DATA_ROOT if custom data_dir provided
323
+ original_root = tp.DATA_ROOT
324
+ tp.DATA_ROOT = Path(data_dir)
325
+
326
+ try:
327
+ mdir = metrics_dir(topic_id)
328
+ rdir = raw_dir(topic_id)
329
+
330
+ predictions_path = mdir / "predictions.jsonl"
331
+ predictions = load_predictions(predictions_path)
332
+
333
+ if not predictions:
334
+ scorecard = generate_scorecard([])
335
+ save_scorecard(scorecard, mdir)
336
+ return scorecard
337
+
338
+ validated_count = 0
339
+ for pred in predictions:
340
+ if pred.get("validated") is not None:
341
+ continue
342
+ pred_week = pred.get("week", "")
343
+ if not pred_week or not is_old_enough(pred_week, weeks_ago):
344
+ continue
345
+
346
+ result = validate_prediction(pred, rdir, weeks_ago)
347
+ if result is not None:
348
+ pred["validated"] = result
349
+ validated_count += 1
350
+
351
+ # Write updated predictions
352
+ save_predictions(predictions, predictions_path)
353
+
354
+ # Generate and save scorecard
355
+ scorecard = generate_scorecard(predictions)
356
+ scorecard_path = save_scorecard(scorecard, mdir)
357
+
358
+ # Print summary
359
+ print(f"Validated {validated_count} predictions")
360
+ print(f"Overall accuracy: {scorecard['accuracy']:.1%}")
361
+ print(f" Correct: {scorecard['correct']}")
362
+ print(f" Incorrect: {scorecard['incorrect']}")
363
+ for ptype, stats in scorecard.get("by_type", {}).items():
364
+ print(f" [{ptype}] {stats['correct']}/{stats['total']} ({stats['accuracy']:.1%})")
365
+ print(f"Scorecard saved to {scorecard_path}")
366
+
367
+ return scorecard
368
+ finally:
369
+ tp.DATA_ROOT = original_root
370
+
371
+
372
+def main(argv: list[str] | None = None) -> dict[str, Any]:
373
+ """CLI entry point."""
374
+ args = parse_args(argv)
375
+ return run_validation(
376
+ topic_id=args.topic,
377
+ weeks_ago=args.weeks_ago,
378
+ data_dir=args.data_dir,
379
+ )
380
+
381
+
382
+if __name__ == "__main__":
383
+ main()
tests/test_hindsight_validation.py
new
+363
@@ -0,0 +1,363 @@
1
+"""Tests for scripts/hindsight_validation.py"""
2
+
3
+from __future__ import annotations
4
+
5
+import json
6
+from datetime import datetime, timedelta
7
+from pathlib import Path
8
+from typing import Any
9
+
10
+import pytest
11
+
12
+from scripts.hindsight_validation import (
13
+ build_repo_set,
14
+ build_repo_stars,
15
+ current_iso_week,
16
+ generate_scorecard,
17
+ is_old_enough,
18
+ iso_week_to_date,
19
+ load_predictions,
20
+ run_validation,
21
+ save_predictions,
22
+ validate_breakout_candidate,
23
+ validate_declining_signal,
24
+ validate_momentum_shift,
25
+ validate_prediction,
26
+ validate_rising_star,
27
+ week_offset,
28
+)
29
+
30
+
31
+def _old_week(weeks_back: int = 5) -> str:
32
+ """Return a week string N weeks in the past."""
33
+ dt = datetime.now() - timedelta(weeks=weeks_back)
34
+ cal = dt.isocalendar()
35
+ return f"{cal[0]}-W{cal[1]:02d}"
36
+
37
+
38
+def _make_raw_data(repos: list[dict[str, Any]], section: str = "trending_repos") -> dict[str, Any]:
39
+ return {section: repos, "new_repos": [] if section != "new_repos" else repos,
40
+ "trending_repos": [] if section != "trending_repos" else repos}
41
+
42
+
43
+def _write_raw(raw_dir: Path, week: str, data: dict[str, Any]) -> None:
44
+ raw_dir.mkdir(parents=True, exist_ok=True)
45
+ with open(raw_dir / f"{week}.json", "w") as f:
46
+ json.dump(data, f)
47
+
48
+
49
+def _write_predictions(metrics_dir: Path, predictions: list[dict[str, Any]]) -> None:
50
+ metrics_dir.mkdir(parents=True, exist_ok=True)
51
+ with open(metrics_dir / "predictions.jsonl", "w") as f:
52
+ for p in predictions:
53
+ f.write(json.dumps(p) + "\n")
54
+
55
+
56
+class TestIsoWeekConversion:
57
+ def test_round_trip(self):
58
+ week = "2026-W21"
59
+ dt = iso_week_to_date(week)
60
+ assert dt.year == 2026
61
+
62
+ def test_week_offset_forward(self):
63
+ result = week_offset("2026-W10", 4)
64
+ assert result == "2026-W14"
65
+
66
+ def test_week_offset_backward(self):
67
+ result = week_offset("2026-W10", -2)
68
+ assert result == "2026-W08"
69
+
70
+ def test_current_iso_week_format(self):
71
+ week = current_iso_week()
72
+ assert len(week.split("-W")) == 2
73
+
74
+
75
+class TestLoadSavePredictions:
76
+ def test_load_missing_file(self, tmp_path: Path):
77
+ result = load_predictions(tmp_path / "nonexistent.jsonl")
78
+ assert result == []
79
+
80
+ def test_round_trip(self, tmp_path: Path):
81
+ preds = [
82
+ {"week": "2026-W21", "repo": "org/a", "prediction": "rising_star",
83
+ "confidence": 0.7, "reason": "test", "validated": None},
84
+ ]
85
+ path = tmp_path / "predictions.jsonl"
86
+ save_predictions(preds, path)
87
+ loaded = load_predictions(path)
88
+ assert loaded == preds
89
+
90
+ def test_empty_lines_ignored(self, tmp_path: Path):
91
+ path = tmp_path / "predictions.jsonl"
92
+ path.write_text('{"week":"2026-W01","repo":"x/y","prediction":"rising_star","confidence":0.5,"reason":"r","validated":null}\n\n')
93
+ loaded = load_predictions(path)
94
+ assert len(loaded) == 1
95
+
96
+
97
+class TestBuildRepoHelpers:
98
+ def test_build_repo_stars(self):
99
+ raw = {"trending_repos": [{"full_name": "a/b", "stars": 100}], "new_repos": []}
100
+ assert build_repo_stars(raw) == {"a/b": 100}
101
+
102
+ def test_build_repo_set(self):
103
+ raw = {"trending_repos": [{"full_name": "a/b", "stars": 100}],
104
+ "new_repos": [{"full_name": "c/d", "stars": 50}]}
105
+ assert build_repo_set(raw) == {"a/b", "c/d"}
106
+
107
+
108
+class TestValidateRisingStar:
109
+ def test_growth_above_threshold(self, tmp_path: Path):
110
+ pred_week = _old_week(5)
111
+ later_week = week_offset(pred_week, 4)
112
+ _write_raw(tmp_path, later_week, _make_raw_data(
113
+ [{"full_name": "org/repo", "stars": 150}]))
114
+ result = validate_rising_star("org/repo", 100, tmp_path, pred_week, 4)
115
+ assert result is True # 50% growth >= 20%
116
+
117
+ def test_growth_below_threshold(self, tmp_path: Path):
118
+ pred_week = _old_week(5)
119
+ later_week = week_offset(pred_week, 4)
120
+ _write_raw(tmp_path, later_week, _make_raw_data(
121
+ [{"full_name": "org/repo", "stars": 110}]))
122
+ result = validate_rising_star("org/repo", 100, tmp_path, pred_week, 4)
123
+ assert result is False
124
+
125
+ def test_no_data_returns_none(self, tmp_path: Path):
126
+ pred_week = _old_week(5)
127
+ result = validate_rising_star("org/repo", 100, tmp_path, pred_week, 4)
128
+ assert result is None
129
+
130
+ def test_zero_stars_baseline(self, tmp_path: Path):
131
+ pred_week = _old_week(5)
132
+ later_week = week_offset(pred_week, 4)
133
+ _write_raw(tmp_path, later_week, _make_raw_data(
134
+ [{"full_name": "org/repo", "stars": 10}]))
135
+ result = validate_rising_star("org/repo", 0, tmp_path, pred_week, 4)
136
+ assert result is True
137
+
138
+
139
+class TestValidateBreakoutCandidate:
140
+ def test_found_in_trending(self, tmp_path: Path):
141
+ pred_week = _old_week(5)
142
+ w2 = week_offset(pred_week, 2)
143
+ _write_raw(tmp_path, w2, _make_raw_data(
144
+ [{"full_name": "org/repo", "stars": 200}], "trending_repos"))
145
+ result = validate_breakout_candidate("org/repo", tmp_path, pred_week, 4)
146
+ assert result is True
147
+
148
+ def test_not_found(self, tmp_path: Path):
149
+ pred_week = _old_week(5)
150
+ w1 = week_offset(pred_week, 1)
151
+ _write_raw(tmp_path, w1, _make_raw_data(
152
+ [{"full_name": "other/repo", "stars": 200}], "trending_repos"))
153
+ result = validate_breakout_candidate("org/repo", tmp_path, pred_week, 4)
154
+ assert result is False
155
+
156
+
157
+class TestValidateMomentumShift:
158
+ def test_continued_growth(self, tmp_path: Path):
159
+ pred_week = _old_week(5)
160
+ w2 = week_offset(pred_week, 2)
161
+ _write_raw(tmp_path, w2, _make_raw_data(
162
+ [{"full_name": "org/repo", "stars": 1200}]))
163
+ result = validate_momentum_shift("org/repo", 1000, tmp_path, pred_week, 4)
164
+ assert result is True
165
+
166
+ def test_declined(self, tmp_path: Path):
167
+ pred_week = _old_week(5)
168
+ w2 = week_offset(pred_week, 2)
169
+ _write_raw(tmp_path, w2, _make_raw_data(
170
+ [{"full_name": "org/repo", "stars": 800}]))
171
+ result = validate_momentum_shift("org/repo", 1000, tmp_path, pred_week, 4)
172
+ assert result is False
173
+
174
+ def test_no_data(self, tmp_path: Path):
175
+ pred_week = _old_week(5)
176
+ result = validate_momentum_shift("org/repo", 1000, tmp_path, pred_week, 4)
177
+ assert result is None
178
+
179
+
180
+class TestValidateDecliningSignal:
181
+ def test_repo_disappeared(self, tmp_path: Path):
182
+ pred_week = _old_week(5)
183
+ for i in range(1, 5):
184
+ w = week_offset(pred_week, i)
185
+ _write_raw(tmp_path, w, _make_raw_data(
186
+ [{"full_name": "other/repo", "stars": 50}]))
187
+ result = validate_declining_signal("org/repo", tmp_path, pred_week, 4)
188
+ assert result is True
189
+
190
+ def test_repo_still_present(self, tmp_path: Path):
191
+ pred_week = _old_week(5)
192
+ for i in range(1, 5):
193
+ w = week_offset(pred_week, i)
194
+ _write_raw(tmp_path, w, _make_raw_data(
195
+ [{"full_name": "org/repo", "stars": 50}]))
196
+ result = validate_declining_signal("org/repo", tmp_path, pred_week, 4)
197
+ assert result is False
198
+
199
+ def test_no_data(self, tmp_path: Path):
200
+ pred_week = _old_week(5)
201
+ result = validate_declining_signal("org/repo", tmp_path, pred_week, 4)
202
+ assert result is None
203
+
204
+
205
+class TestIsOldEnough:
206
+ def test_old_prediction(self):
207
+ week = _old_week(6)
208
+ assert is_old_enough(week, 4) is True
209
+
210
+ def test_recent_prediction(self):
211
+ week = current_iso_week()
212
+ assert is_old_enough(week, 4) is False
213
+
214
+ def test_invalid_week(self):
215
+ assert is_old_enough("", 4) is False
216
+ assert is_old_enough("not-a-week", 4) is False
217
+
218
+
219
+class TestGenerateScorecard:
220
+ def test_empty(self):
221
+ sc = generate_scorecard([])
222
+ assert sc["total_validated"] == 0
223
+ assert sc["accuracy"] == 0.0
224
+
225
+ def test_mixed_results(self):
226
+ preds = [
227
+ {"prediction": "rising_star", "validated": True},
228
+ {"prediction": "rising_star", "validated": False},
229
+ {"prediction": "momentum_shift", "validated": True},
230
+ {"prediction": "rising_star", "validated": None}, # not counted
231
+ ]
232
+ sc = generate_scorecard(preds)
233
+ assert sc["total_validated"] == 3
234
+ assert sc["correct"] == 2
235
+ assert sc["incorrect"] == 1
236
+ assert sc["accuracy"] == pytest.approx(2 / 3, abs=0.001)
237
+ assert sc["by_type"]["rising_star"]["total"] == 2
238
+ assert sc["by_type"]["rising_star"]["correct"] == 1
239
+ assert sc["by_type"]["momentum_shift"]["total"] == 1
240
+
241
+
242
+class TestValidatePrediction:
243
+ def test_unknown_type(self, tmp_path: Path):
244
+ pred = {"week": _old_week(5), "repo": "org/repo", "prediction": "unknown_type"}
245
+ result = validate_prediction(pred, tmp_path, 4)
246
+ assert result is None
247
+
248
+ def test_missing_repo(self, tmp_path: Path):
249
+ pred = {"week": _old_week(5), "repo": "", "prediction": "rising_star"}
250
+ result = validate_prediction(pred, tmp_path, 4)
251
+ assert result is None
252
+
253
+ def test_emerging_topic_uses_declining_logic(self, tmp_path: Path):
254
+ pred_week = _old_week(5)
255
+ pred = {"week": pred_week, "repo": "org/repo", "prediction": "emerging_topic"}
256
+ for i in range(1, 5):
257
+ w = week_offset(pred_week, i)
258
+ _write_raw(tmp_path, w, _make_raw_data(
259
+ [{"full_name": "other/repo", "stars": 50}]))
260
+ result = validate_prediction(pred, tmp_path, 4)
261
+ assert result is True
262
+
263
+
264
+class TestRunValidation:
265
+ def test_full_run(self, tmp_path: Path):
266
+ pred_week = _old_week(5)
267
+ raw = tmp_path / "raw"
268
+ metrics = tmp_path / "metrics"
269
+
270
+ # Write prediction-week raw data
271
+ _write_raw(raw, pred_week, _make_raw_data(
272
+ [{"full_name": "org/star", "stars": 100}]))
273
+
274
+ # Write later-week raw data with growth
275
+ later = week_offset(pred_week, 4)
276
+ _write_raw(raw, later, _make_raw_data(
277
+ [{"full_name": "org/star", "stars": 200}]))
278
+
279
+ # Write predictions
280
+ preds = [
281
+ {"week": pred_week, "repo": "org/star", "prediction": "rising_star",
282
+ "confidence": 0.7, "reason": "test", "validated": None},
283
+ ]
284
+ _write_predictions(metrics, preds)
285
+
286
+ scorecard = run_validation(topic_id=None, weeks_ago=4, data_dir=str(tmp_path))
287
+
288
+ assert scorecard["total_validated"] == 1
289
+ assert scorecard["correct"] == 1
290
+ assert scorecard["accuracy"] == 1.0
291
+
292
+ # Check predictions file was updated
293
+ updated = load_predictions(metrics / "predictions.jsonl")
294
+ assert updated[0]["validated"] is True
295
+
296
+ # Check scorecard file was written
297
+ scorecards = list((metrics / "scorecards").glob("*-scorecard.json"))
298
+ assert len(scorecards) == 1
299
+
300
+ def test_skips_recent_predictions(self, tmp_path: Path):
301
+ raw = tmp_path / "raw"
302
+ metrics = tmp_path / "metrics"
303
+ raw.mkdir(parents=True, exist_ok=True)
304
+
305
+ recent_week = current_iso_week()
306
+ preds = [
307
+ {"week": recent_week, "repo": "org/new", "prediction": "rising_star",
308
+ "confidence": 0.7, "reason": "too new", "validated": None},
309
+ ]
310
+ _write_predictions(metrics, preds)
311
+
312
+ scorecard = run_validation(topic_id=None, weeks_ago=4, data_dir=str(tmp_path))
313
+ assert scorecard["total_validated"] == 0
314
+
315
+ # Prediction should remain unvalidated
316
+ updated = load_predictions(metrics / "predictions.jsonl")
317
+ assert updated[0]["validated"] is None
318
+
319
+ def test_empty_predictions(self, tmp_path: Path):
320
+ metrics = tmp_path / "metrics"
321
+ metrics.mkdir(parents=True, exist_ok=True)
322
+ scorecard = run_validation(topic_id=None, weeks_ago=4, data_dir=str(tmp_path))
323
+ assert scorecard["total_validated"] == 0
324
+
325
+ def test_topic_path_resolution(self, tmp_path: Path):
326
+ pred_week = _old_week(5)
327
+ topic = "ai-ml"
328
+ raw = tmp_path / "raw" / topic
329
+ metrics = tmp_path / "metrics" / topic
330
+
331
+ _write_raw(raw, pred_week, _make_raw_data(
332
+ [{"full_name": "org/ai", "stars": 100}]))
333
+ later = week_offset(pred_week, 4)
334
+ _write_raw(raw, later, _make_raw_data(
335
+ [{"full_name": "org/ai", "stars": 130}]))
336
+
337
+ preds = [
338
+ {"week": pred_week, "repo": "org/ai", "prediction": "rising_star",
339
+ "confidence": 0.6, "reason": "test", "validated": None},
340
+ ]
341
+ _write_predictions(metrics, preds)
342
+
343
+ scorecard = run_validation(topic_id=topic, weeks_ago=4, data_dir=str(tmp_path))
344
+ assert scorecard["total_validated"] == 1
345
+ # 30% growth >= 20% threshold
346
+ assert scorecard["correct"] == 1
347
+
348
+ def test_already_validated_skipped(self, tmp_path: Path):
349
+ raw = tmp_path / "raw"
350
+ metrics = tmp_path / "metrics"
351
+ raw.mkdir(parents=True, exist_ok=True)
352
+
353
+ pred_week = _old_week(5)
354
+ preds = [
355
+ {"week": pred_week, "repo": "org/done", "prediction": "rising_star",
356
+ "confidence": 0.7, "reason": "already done", "validated": True},
357
+ ]
358
+ _write_predictions(metrics, preds)
359
+
360
+ scorecard = run_validation(topic_id=None, weeks_ago=4, data_dir=str(tmp_path))
361
+ # Already validated, so it counts in scorecard but wasn't re-processed
362
+ assert scorecard["total_validated"] == 1
363
+ assert scorecard["correct"] == 1