| 1 | """Tests for the prediction ledger module.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import json |
| 6 | from pathlib import Path |
| 7 | from unittest.mock import patch |
| 8 | |
| 9 | import pytest |
| 10 | |
| 11 | from scripts.prediction_ledger import ( |
| 12 | MAX_PREDICTIONS, |
| 13 | MIN_PREDICTIONS, |
| 14 | PREDICTION_TYPES, |
| 15 | append_predictions, |
| 16 | build_repo_index, |
| 17 | classify_prediction, |
| 18 | extract_repos_from_summary, |
| 19 | extract_week, |
| 20 | generate_predictions, |
| 21 | infer_raw_path, |
| 22 | main, |
| 23 | parse_summary, |
| 24 | score_breakout_candidate, |
| 25 | score_momentum_shift, |
| 26 | score_rising_star, |
| 27 | ) |
| 28 | |
| 29 | SAMPLE_SUMMARY = """\ |
| 30 | --- |
| 31 | title: "Week 21, 2026 Analysis" |
| 32 | date: 2026-05-18T12:07:20.778+02:00 |
| 33 | week: "2026-W21" |
| 34 | year: 2026 |
| 35 | tags: [ai, agents] |
| 36 | top_repo: "vercel-labs/zero" |
| 37 | quality_score: 76 |
| 38 | summary: "Strong week for agent tooling." |
| 39 | --- |
| 40 | |
| 41 | ## This Week's Trends |
| 42 | |
| 43 | [vercel-labs/zero](https://github.com/vercel-labs/zero) led this week as a standout new AI infrastructure repo. |
| 44 | Also notable: [org/rising-repo](https://github.com/org/rising-repo) and |
| 45 | [bigcorp/established](https://github.com/bigcorp/established). |
| 46 | """ |
| 47 | |
| 48 | SAMPLE_RAW = { |
| 49 | "week": "2026-W21", |
| 50 | "crawled_at": "2026-05-18T08:54:09Z", |
| 51 | "new_repos": [ |
| 52 | { |
| 53 | "name": "zero", |
| 54 | "owner": "vercel-labs", |
| 55 | "full_name": "vercel-labs/zero", |
| 56 | "description": "Agent infra", |
| 57 | "language": "TypeScript", |
| 58 | "stars": 2500, |
| 59 | "forks": 150, |
| 60 | "created_at": "2026-05-10T00:00:00Z", |
| 61 | "topics": ["ai", "agents"], |
| 62 | "license": "MIT", |
| 63 | "url": "https://github.com/vercel-labs/zero", |
| 64 | }, |
| 65 | { |
| 66 | "name": "rising-repo", |
| 67 | "owner": "org", |
| 68 | "full_name": "org/rising-repo", |
| 69 | "description": "Hot new project", |
| 70 | "language": "Python", |
| 71 | "stars": 800, |
| 72 | "forks": 120, |
| 73 | "created_at": "2026-05-12T00:00:00Z", |
| 74 | "topics": ["ml"], |
| 75 | "license": "Apache-2.0", |
| 76 | "url": "https://github.com/org/rising-repo", |
| 77 | }, |
| 78 | ], |
| 79 | "trending_repos": [ |
| 80 | { |
| 81 | "name": "established", |
| 82 | "owner": "bigcorp", |
| 83 | "full_name": "bigcorp/established", |
| 84 | "description": "Major framework", |
| 85 | "language": "JavaScript", |
| 86 | "stars": 50000, |
| 87 | "forks": 8000, |
| 88 | "created_at": "2020-01-01T00:00:00Z", |
| 89 | "topics": ["framework"], |
| 90 | "license": "MIT", |
| 91 | "url": "https://github.com/bigcorp/established", |
| 92 | }, |
| 93 | ], |
| 94 | "signals": {"top_topics": ["ai", "agents"]}, |
| 95 | "metadata": {"api_calls_used": 10, "rate_limit_remaining": 4990}, |
| 96 | } |
| 97 | |
| 98 | |
| 99 | class TestExtractWeek: |
| 100 | def test_extracts_from_filename(self): |
| 101 | assert extract_week("2026-W21-summary.md") == "2026-W21" |
| 102 | |
| 103 | def test_extracts_from_text(self): |
| 104 | assert extract_week("week: 2025-W03") == "2025-W03" |
| 105 | |
| 106 | def test_returns_none_for_no_match(self): |
| 107 | assert extract_week("no week here") is None |
| 108 | |
| 109 | |
| 110 | class TestParseSummary: |
| 111 | def test_parses_frontmatter(self): |
| 112 | result = parse_summary(SAMPLE_SUMMARY) |
| 113 | assert result["frontmatter"]["week"] == "2026-W21" |
| 114 | assert result["frontmatter"]["top_repo"] == "vercel-labs/zero" |
| 115 | |
| 116 | def test_parses_body(self): |
| 117 | result = parse_summary(SAMPLE_SUMMARY) |
| 118 | assert "This Week's Trends" in result["body"] |
| 119 | |
| 120 | def test_handles_no_frontmatter(self): |
| 121 | result = parse_summary("# Just a header\nSome content.") |
| 122 | assert result["frontmatter"] == {} |
| 123 | assert "Just a header" in result["body"] |
| 124 | |
| 125 | |
| 126 | class TestExtractRepos: |
| 127 | def test_extracts_repo_links(self): |
| 128 | body = parse_summary(SAMPLE_SUMMARY)["body"] |
| 129 | repos = extract_repos_from_summary(body) |
| 130 | assert "vercel-labs/zero" in repos |
| 131 | assert "org/rising-repo" in repos |
| 132 | assert "bigcorp/established" in repos |
| 133 | |
| 134 | def test_deduplicates(self): |
| 135 | text = "[a/b](https://github.com/a/b) and [a/b](https://github.com/a/b)" |
| 136 | repos = extract_repos_from_summary(text) |
| 137 | assert repos == ["a/b"] |
| 138 | |
| 139 | def test_empty_for_no_links(self): |
| 140 | assert extract_repos_from_summary("no repos here") == [] |
| 141 | |
| 142 | |
| 143 | class TestBuildRepoIndex: |
| 144 | def test_indexes_by_full_name(self): |
| 145 | index = build_repo_index(SAMPLE_RAW) |
| 146 | assert "vercel-labs/zero" in index |
| 147 | assert "bigcorp/established" in index |
| 148 | |
| 149 | def test_marks_source(self): |
| 150 | index = build_repo_index(SAMPLE_RAW) |
| 151 | assert index["vercel-labs/zero"]["_source"] == "new_repos" |
| 152 | assert index["bigcorp/established"]["_source"] == "trending_repos" |
| 153 | |
| 154 | def test_empty_data(self): |
| 155 | assert build_repo_index({}) == {} |
| 156 | |
| 157 | |
| 158 | class TestScoring: |
| 159 | def test_rising_star_high_stars_new(self): |
| 160 | repo = {"stars": 3000, "_source": "new_repos"} |
| 161 | score = score_rising_star(repo) |
| 162 | assert 0.5 <= score <= 0.9 |
| 163 | |
| 164 | def test_rising_star_low_stars(self): |
| 165 | repo = {"stars": 10, "_source": "new_repos"} |
| 166 | score = score_rising_star(repo) |
| 167 | assert score < 0.4 |
| 168 | |
| 169 | def test_breakout_high_fork_ratio(self): |
| 170 | repo = {"stars": 500, "forks": 100, "_source": "new_repos"} |
| 171 | score = score_breakout_candidate(repo) |
| 172 | assert score >= 0.4 |
| 173 | |
| 174 | def test_momentum_shift_trending_big(self): |
| 175 | repo = {"stars": 50000, "_source": "trending_repos"} |
| 176 | score = score_momentum_shift(repo) |
| 177 | assert score >= 0.5 |
| 178 | |
| 179 | def test_momentum_shift_not_trending(self): |
| 180 | repo = {"stars": 50000, "_source": "new_repos"} |
| 181 | score = score_momentum_shift(repo) |
| 182 | assert score < 0.3 |
| 183 | |
| 184 | |
| 185 | class TestClassifyPrediction: |
| 186 | def test_returns_valid_type(self): |
| 187 | repo = {"stars": 3000, "forks": 100, "_source": "new_repos"} |
| 188 | pred_type, confidence, reason = classify_prediction(repo) |
| 189 | assert pred_type in PREDICTION_TYPES |
| 190 | assert 0.0 <= confidence <= 1.0 |
| 191 | assert len(reason) > 0 |
| 192 | |
| 193 | def test_new_high_star_is_rising_star(self): |
| 194 | repo = {"stars": 5000, "forks": 50, "_source": "new_repos"} |
| 195 | pred_type, _, _ = classify_prediction(repo) |
| 196 | assert pred_type == "rising_star" |
| 197 | |
| 198 | def test_trending_established_is_momentum(self): |
| 199 | repo = {"stars": 50000, "forks": 1000, "_source": "trending_repos"} |
| 200 | pred_type, _, _ = classify_prediction(repo) |
| 201 | assert pred_type == "momentum_shift" |
| 202 | |
| 203 | |
| 204 | class TestGeneratePredictions: |
| 205 | def test_generates_predictions(self): |
| 206 | preds = generate_predictions(SAMPLE_SUMMARY, SAMPLE_RAW, "2026-W21") |
| 207 | assert MIN_PREDICTIONS <= len(preds) <= MAX_PREDICTIONS |
| 208 | |
| 209 | def test_prediction_structure(self): |
| 210 | preds = generate_predictions(SAMPLE_SUMMARY, SAMPLE_RAW, "2026-W21") |
| 211 | for p in preds: |
| 212 | assert p["week"] == "2026-W21" |
| 213 | assert p["prediction"] in PREDICTION_TYPES |
| 214 | assert 0.0 <= p["confidence"] <= 1.0 |
| 215 | assert p["validated"] is None |
| 216 | assert "repo" in p |
| 217 | assert "reason" in p |
| 218 | |
| 219 | def test_sorted_by_confidence(self): |
| 220 | preds = generate_predictions(SAMPLE_SUMMARY, SAMPLE_RAW, "2026-W21") |
| 221 | confidences = [p["confidence"] for p in preds] |
| 222 | assert confidences == sorted(confidences, reverse=True) |
| 223 | |
| 224 | |
| 225 | class TestAppendPredictions: |
| 226 | def test_appends_to_file(self, tmp_path): |
| 227 | output = tmp_path / "predictions.jsonl" |
| 228 | preds = [ |
| 229 | { |
| 230 | "week": "2026-W21", |
| 231 | "repo": "a/b", |
| 232 | "prediction": "rising_star", |
| 233 | "confidence": 0.7, |
| 234 | "reason": "test", |
| 235 | "validated": None, |
| 236 | } |
| 237 | ] |
| 238 | append_predictions(preds, output) |
| 239 | lines = output.read_text().strip().splitlines() |
| 240 | assert len(lines) == 1 |
| 241 | assert json.loads(lines[0])["repo"] == "a/b" |
| 242 | |
| 243 | def test_appends_multiple_calls(self, tmp_path): |
| 244 | output = tmp_path / "predictions.jsonl" |
| 245 | pred = { |
| 246 | "week": "2026-W21", |
| 247 | "repo": "x/y", |
| 248 | "prediction": "rising_star", |
| 249 | "confidence": 0.5, |
| 250 | "reason": "r", |
| 251 | "validated": None, |
| 252 | } |
| 253 | append_predictions([pred], output) |
| 254 | append_predictions([pred], output) |
| 255 | lines = output.read_text().strip().splitlines() |
| 256 | assert len(lines) == 2 |
| 257 | |
| 258 | def test_creates_parent_dirs(self, tmp_path): |
| 259 | output = tmp_path / "nested" / "dir" / "predictions.jsonl" |
| 260 | append_predictions([], output) |
| 261 | assert output.parent.exists() |
| 262 | |
| 263 | |
| 264 | class TestInferRawPath: |
| 265 | def test_infers_from_summary(self): |
| 266 | summary = Path("data/analyzed/2026-W21-summary.md") |
| 267 | raw = infer_raw_path(summary, None) |
| 268 | assert raw == Path("data/raw/2026-W21.json") |
| 269 | |
| 270 | def test_infers_with_topic(self): |
| 271 | summary = Path("data/analyzed/ai-ml/2026-W21-summary.md") |
| 272 | raw = infer_raw_path(summary, "ai-ml") |
| 273 | assert raw == Path("data/raw/ai-ml/2026-W21.json") |
| 274 | |
| 275 | def test_raises_on_no_week(self): |
| 276 | with pytest.raises(ValueError): |
| 277 | infer_raw_path(Path("bad-name.md"), None) |
| 278 | |
| 279 | |
| 280 | class TestMain: |
| 281 | def test_end_to_end(self, tmp_path): |
| 282 | # Set up files |
| 283 | summary_path = tmp_path / "analyzed" / "2026-W21-summary.md" |
| 284 | summary_path.parent.mkdir(parents=True) |
| 285 | summary_path.write_text(SAMPLE_SUMMARY) |
| 286 | |
| 287 | raw_path = tmp_path / "raw" / "2026-W21.json" |
| 288 | raw_path.parent.mkdir(parents=True) |
| 289 | raw_path.write_text(json.dumps(SAMPLE_RAW)) |
| 290 | |
| 291 | metrics_path = tmp_path / "metrics" |
| 292 | |
| 293 | with patch("scripts.prediction_ledger.metrics_dir", return_value=metrics_path): |
| 294 | preds = main( |
| 295 | [ |
| 296 | "--input", |
| 297 | str(summary_path), |
| 298 | "--raw", |
| 299 | str(raw_path), |
| 300 | ] |
| 301 | ) |
| 302 | |
| 303 | assert len(preds) >= MIN_PREDICTIONS |
| 304 | output_file = metrics_path / "predictions.jsonl" |
| 305 | assert output_file.exists() |
| 306 | lines = output_file.read_text().strip().splitlines() |
| 307 | assert len(lines) == len(preds) |