feat: implement hindsight validation script (#65) (#104)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 19, 2026 at 16:03 UTC
017d16a7640ad4096953de3e6aefd14244b8c9d5
2 files changed
+9
-35
scripts/hindsight_validation.py
+1
-18
@@ -119,16 +119,6 @@ def build_repo_set(raw_data: dict[str, Any]) -> set[str]:
119
return repos
120
121
122
-def find_subsequent_weeks(raw_directory: Path, start_week: str, count: int) -> list[str]:
123
- """Find up to `count` weeks of raw data after start_week."""
124
- weeks = []
125
- for i in range(1, count + 1):
126
- w = week_offset(start_week, i)
127
- if (raw_directory / f"{w}.json").exists():
128
- weeks.append(w)
129
- return weeks
130
-
131
-
122
def validate_rising_star(
123
repo: str,
124
prediction_stars: int,
@@ -148,7 +138,7 @@ def validate_rising_star(
138
return current_stars > 0
139
growth = (current_stars - prediction_stars) / prediction_stars
140
return growth >= 0.20
151
- return None # no data to validate
141
+ return None
142
143
144
def validate_breakout_candidate(
@@ -192,7 +182,6 @@ def validate_momentum_shift(
182
183
if len(star_history) < 2:
184
return None
195
- # Check if direction is consistent (non-decreasing)
185
return star_history[-1] >= star_history[0]
186
187
@@ -216,7 +205,6 @@ def validate_declining_signal(
205
206
if checked_count == 0:
207
return None
219
- # Validated if repo is absent from majority of subsequent crawls
208
return found_count <= checked_count // 2
209
210
@@ -233,7 +221,6 @@ def validate_prediction(
221
if not repo or not prediction_week:
222
return None
223
236
- # Load prediction week data to get baseline stars
224
pred_raw = load_raw_week(raw_directory, prediction_week)
225
prediction_stars = 0
226
if pred_raw:
@@ -319,7 +306,6 @@ def run_validation(
306
"""Main validation logic. Returns the scorecard."""
307
import scripts.topic_paths as tp
308
322
- # Override DATA_ROOT if custom data_dir provided
309
original_root = tp.DATA_ROOT
310
tp.DATA_ROOT = Path(data_dir)
311
@@ -348,14 +334,11 @@ def run_validation(
334
pred["validated"] = result
335
validated_count += 1
336
351
- # Write updated predictions
337
save_predictions(predictions, predictions_path)
338
354
- # Generate and save scorecard
339
scorecard = generate_scorecard(predictions)
340
scorecard_path = save_scorecard(scorecard, mdir)
341
358
- # Print summary
342
print(f"Validated {validated_count} predictions")
343
print(f"Overall accuracy: {scorecard['accuracy']:.1%}")
344
print(f" Correct: {scorecard['correct']}")
tests/test_hindsight_validation.py
+8
-17
@@ -36,8 +36,10 @@ def _old_week(weeks_back: int = 5) -> str:
36
37
38
def _make_raw_data(repos: list[dict[str, Any]], section: str = "trending_repos") -> dict[str, Any]:
39
- return {section: repos, "new_repos": [] if section != "new_repos" else repos,
40
- "trending_repos": [] if section != "trending_repos" else repos}
39
+ return {
40
+ "new_repos": repos if section == "new_repos" else [],
41
+ "trending_repos": repos if section == "trending_repos" else [],
42
+ }
43
44
45
def _write_raw(raw_dir: Path, week: str, data: dict[str, Any]) -> None:
@@ -60,12 +62,10 @@ class TestIsoWeekConversion:
62
assert dt.year == 2026
63
64
def test_week_offset_forward(self):
63
- result = week_offset("2026-W10", 4)
64
- assert result == "2026-W14"
65
+ assert week_offset("2026-W10", 4) == "2026-W14"
66
67
def test_week_offset_backward(self):
67
- result = week_offset("2026-W10", -2)
68
- assert result == "2026-W08"
68
+ assert week_offset("2026-W10", -2) == "2026-W08"
69
70
def test_current_iso_week_format(self):
71
week = current_iso_week()
@@ -112,7 +112,7 @@ class TestValidateRisingStar:
112
_write_raw(tmp_path, later_week, _make_raw_data(
113
[{"full_name": "org/repo", "stars": 150}]))
114
result = validate_rising_star("org/repo", 100, tmp_path, pred_week, 4)
115
- assert result is True # 50% growth >= 20%
115
+ assert result is True
116
117
def test_growth_below_threshold(self, tmp_path: Path):
118
pred_week = _old_week(5)
@@ -227,7 +227,7 @@ class TestGenerateScorecard:
227
{"prediction": "rising_star", "validated": True},
228
{"prediction": "rising_star", "validated": False},
229
{"prediction": "momentum_shift", "validated": True},
230
- {"prediction": "rising_star", "validated": None}, # not counted
230
+ {"prediction": "rising_star", "validated": None},
231
]
232
sc = generate_scorecard(preds)
233
assert sc["total_validated"] == 3
@@ -267,16 +267,12 @@ class TestRunValidation:
267
raw = tmp_path / "raw"
268
metrics = tmp_path / "metrics"
269
270
- # Write prediction-week raw data
270
_write_raw(raw, pred_week, _make_raw_data(
271
[{"full_name": "org/star", "stars": 100}]))
273
-
274
- # Write later-week raw data with growth
272
later = week_offset(pred_week, 4)
273
_write_raw(raw, later, _make_raw_data(
274
[{"full_name": "org/star", "stars": 200}]))
275
279
- # Write predictions
276
preds = [
277
{"week": pred_week, "repo": "org/star", "prediction": "rising_star",
278
"confidence": 0.7, "reason": "test", "validated": None},
@@ -289,11 +285,9 @@ class TestRunValidation:
285
assert scorecard["correct"] == 1
286
assert scorecard["accuracy"] == 1.0
287
292
- # Check predictions file was updated
288
updated = load_predictions(metrics / "predictions.jsonl")
289
assert updated[0]["validated"] is True
290
296
- # Check scorecard file was written
291
scorecards = list((metrics / "scorecards").glob("*-scorecard.json"))
292
assert len(scorecards) == 1
293
@@ -312,7 +306,6 @@ class TestRunValidation:
306
scorecard = run_validation(topic_id=None, weeks_ago=4, data_dir=str(tmp_path))
307
assert scorecard["total_validated"] == 0
308
315
- # Prediction should remain unvalidated
309
updated = load_predictions(metrics / "predictions.jsonl")
310
assert updated[0]["validated"] is None
311
@@ -342,7 +335,6 @@ class TestRunValidation:
335
336
scorecard = run_validation(topic_id=topic, weeks_ago=4, data_dir=str(tmp_path))
337
assert scorecard["total_validated"] == 1
345
- # 30% growth >= 20% threshold
338
assert scorecard["correct"] == 1
339
340
def test_already_validated_skipped(self, tmp_path: Path):
@@ -358,6 +350,5 @@ class TestRunValidation:
350
_write_predictions(metrics, preds)
351
352
scorecard = run_validation(topic_id=None, weeks_ago=4, data_dir=str(tmp_path))
361
- # Already validated, so it counts in scorecard but wasn't re-processed
353
assert scorecard["total_validated"] == 1
354
assert scorecard["correct"] == 1