@cryptotaxi247 / SquadScope-1
Code Commits Issues Pulls Sessions CI/CD
main
main
py 222 lines 6.74 KB
Raw
1 from __future__ import annotations
2
3 import json
4 import shutil
5 from pathlib import Path
6
7 from scripts import validate_predictions
8
9 WORKSPACE_ROOT = Path(__file__).resolve().parent / "_workspace_validate_predictions"
10
11
12 def teardown_module() -> None:
13 shutil.rmtree(WORKSPACE_ROOT, ignore_errors=True)
14
15
16 def prepare_workspace(name: str) -> Path:
17 workspace = WORKSPACE_ROOT / name
18 if workspace.exists():
19 shutil.rmtree(workspace)
20 workspace.mkdir(parents=True)
21 return workspace
22
23
24 def write_file(path: Path, content: str) -> None:
25 path.parent.mkdir(parents=True, exist_ok=True)
26 path.write_text(content, encoding="utf-8")
27
28
29 def write_json(path: Path, payload: dict) -> None:
30 path.parent.mkdir(parents=True, exist_ok=True)
31 path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
32
33
34 WEEKLY_SUMMARY = """---
35 week: 2026-W21
36 generated_at: 2026-06-01T00:00:00Z
37 category: dev-tools
38 tags:
39 - agents
40 summary: Validation fixture.
41 quality_score: 74
42 predictions:
43 - repo: acme/launchpad
44 claim_type: signal
45 direction: up
46 confidence: 0.8
47 - repo: acme/flashy-kit
48 claim_type: noise
49 direction: flat
50 confidence: 0.7
51 ---
52
53 ## Signal & Noise
54
55 The durable signal this week is [acme/launchpad](https://github.com/acme/launchpad), which is finding repeat users.
56
57 The noise this week is [acme/flashy-kit](https://github.com/acme/flashy-kit), which looks more like a demo spike than a habit loop.
58
59 ## Blind Spots
60
61 Small teams still need [acme/gap-tool](https://github.com/acme/gap-tool), but the category is not mature yet.
62 """
63
64
65 COMBINED_SUMMARY = """---
66 week: 2026-W22
67 generated_at: 2026-06-01T00:00:00Z
68 category: dev-tools
69 tags:
70 - agents
71 summary: Second fixture.
72 quality_score: 81
73 ---
74
75 ## Signal & Noise
76
77 The durable signal this week is [acme/steady-core](https://github.com/acme/steady-core), which has a real operator workflow behind it.
78
79 The noise this week is [acme/flashy-kit](https://github.com/acme/flashy-kit), which is already cooling.
80
81 ## Blind Spots
82
83 Nobody has fully solved [acme/gap-tool](https://github.com/acme/gap-tool) yet.
84 """
85
86
87 RAW_W21 = {
88 "new_repos": [
89 {"full_name": "acme/launchpad", "stars": 100},
90 {"full_name": "acme/flashy-kit", "stars": 100},
91 {"full_name": "acme/gap-tool", "stars": 80},
92 ],
93 "trending_repos": [],
94 }
95
96 RAW_W22 = {
97 "new_repos": [
98 {"full_name": "acme/launchpad", "stars": 130},
99 {"full_name": "acme/gap-tool", "stars": 100},
100 ],
101 "trending_repos": [
102 {"full_name": "acme/steady-core", "stars": 90},
103 ],
104 }
105
106
107 def test_infer_predictions_from_current_summary_patterns() -> None:
108 summary_path = Path("data/analyzed/2026-W23-summary.md")
109 predictions = validate_predictions.load_summary_predictions(summary_path)
110 claims = {(prediction.claim, prediction.repo) for prediction in predictions}
111
112 assert ("signal", "duncatzat/vigils") in claims
113 assert ("signal", "openai/role-specific-plugins") in claims
114 assert ("noise", "pewdiepie-archdaemon/odysseus") in claims
115
116
117 def test_frontmatter_predictions_use_explicit_claim_type() -> None:
118 workspace = prepare_workspace("frontmatter-claim-type")
119 summary_path = workspace / "2026-W21-summary.md"
120 summary_text = WEEKLY_SUMMARY.replace(
121 " - repo: acme/launchpad\n claim_type: signal\n direction: up\n confidence: 0.8",
122 " - repo: acme/launchpad\n claim_type: gap\n direction: up\n confidence: 0.8",
123 )
124 write_file(summary_path, summary_text)
125
126 predictions = validate_predictions.load_summary_predictions(summary_path)
127 indexed = {prediction.repo: prediction for prediction in predictions}
128
129 assert indexed["acme/launchpad"].source == "frontmatter"
130 assert indexed["acme/launchpad"].claim == "gap"
131 assert indexed["acme/launchpad"].direction == "up"
132 assert indexed["acme/flashy-kit"].claim == "noise"
133
134
135 def test_missing_baseline_repo_becomes_insufficient_evidence() -> None:
136 workspace = prepare_workspace("missing-baseline")
137 raw_dir = workspace / "raw"
138
139 prediction = validate_predictions.Prediction(
140 week="2026-W21",
141 repo="acme/missing-baseline",
142 claim="signal",
143 direction="up",
144 confidence=0.8,
145 source="frontmatter",
146 source_path="fixture.md",
147 )
148
149 write_json(raw_dir / "2026-W21.json", RAW_W21)
150 write_json(raw_dir / "2026-W22.json", RAW_W22)
151
152 result = validate_predictions.evaluate_prediction(prediction, raw_dir, weeks_ahead=4)
153
154 assert result.verdict == "insufficient_evidence"
155 assert result.baseline_stars is None
156 assert result.observed_stars is None
157 assert "prediction-week crawl" in result.note
158
159
160 def test_missing_observed_repo_becomes_insufficient_evidence() -> None:
161 workspace = prepare_workspace("missing-observed")
162 raw_dir = workspace / "raw"
163
164 prediction = validate_predictions.Prediction(
165 week="2026-W21",
166 repo="acme/flashy-kit",
167 claim="noise",
168 direction="flat",
169 confidence=0.7,
170 source="frontmatter",
171 source_path="fixture.md",
172 )
173
174 write_json(raw_dir / "2026-W21.json", RAW_W21)
175 write_json(raw_dir / "2026-W22.json", RAW_W22)
176
177 result = validate_predictions.evaluate_prediction(prediction, raw_dir, weeks_ahead=4)
178
179 assert result.verdict == "insufficient_evidence"
180 assert result.baseline_stars == 100
181 assert result.observed_stars is None
182 assert "later crawl payload" in result.note
183
184
185 def test_run_validation_writes_markdown_and_json_scorecards() -> None:
186 workspace = prepare_workspace("run-validation")
187 analyzed_dir = workspace / "analyzed"
188 raw_dir = workspace / "raw"
189 metrics_dir = workspace / "metrics"
190 scorecard_dir = workspace / "scorecards"
191
192 write_file(analyzed_dir / "2026-W21-summary.md", WEEKLY_SUMMARY)
193 write_file(analyzed_dir / "2026-W22-summary.md", COMBINED_SUMMARY)
194 write_json(raw_dir / "2026-W21.json", RAW_W21)
195 write_json(raw_dir / "2026-W22.json", RAW_W22)
196
197 summary = validate_predictions.run_validation(
198 analyzed_dir=analyzed_dir,
199 raw_dir=raw_dir,
200 metrics_dir=metrics_dir,
201 scorecard_dir=scorecard_dir,
202 report_week="2026-W23",
203 )
204
205 assert summary.week == "2026-W23"
206 assert summary.total_predictions == 5
207 assert summary.validated == 1
208 assert summary.correct == 1
209 assert summary.incorrect == 0
210 assert summary.quality_trend["count"] == 2
211
212 markdown_path = scorecard_dir / "2026-W23.md"
213 json_path = metrics_dir / "scorecards" / "2026-W23-scorecard.json"
214
215 assert markdown_path.exists()
216 assert json_path.exists()
217 assert "Prediction Registry Format" in markdown_path.read_text(encoding="utf-8")
218
219 payload = json.loads(json_path.read_text(encoding="utf-8"))
220 assert payload["validated"] == 1
221 assert payload["total_validated"] == 1
222 assert payload["accuracy"] == 1.0