@cryptotaxi247 / SquadScope-1 / commits / 0f4bb90

feat: wire prediction scorecard into reskill (#67) (#105)

* test: end-to-end integration test with ai-ml topic (#73) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * feat: wire prediction scorecard into reskill (#67) - Add scripts/load_scorecard.py to load and summarize recent scorecards - Add prompts/reskill-scorecard.md template for scorecard-driven reskill - Update reskill.py with --scorecard, --scorecard-count, --topic flags - Add {{SCORECARD}} placeholder to prompts/reskill.md - Add tests/test_load_scorecard.py (14 tests) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Juan Manuel Servera committed May 19, 2026 at 16:06 UTC 0f4bb907ba4fda4e45d970f0396f58f8f58eee52
6 files changed +669
prompts/reskill-scorecard.md new
+8
@@ -0,0 +1,8 @@
1 +You are reviewing prediction accuracy for the {topic_name} topic.
2 +
3 +{scorecard_summary}
4 +
5 +Based on the above performance data, suggest specific adjustments to the topic's wisdom file to improve future predictions. Focus on:
6 +1. Patterns in incorrect predictions — what signals were misleading?
7 +2. Patterns in correct predictions — what signals are reliable?
8 +3. Specific threshold or weight changes to recommend
prompts/reskill.md
+4
@@ -29,6 +29,10 @@ Your job is to review recent analysis output, calibrate the analyst's judgment,
29
30 {{SNAPSHOT_CONTEXT}}
31
32 +### Prediction scorecard
33 +
34 +{{SCORECARD}}
35 +
36 ## Objective
37
38 Write the full contents of `{{OUTPUT_PATH}}` as a markdown reskill report.
scripts/load_scorecard.py new
+140
@@ -0,0 +1,140 @@
1 +#!/usr/bin/env python3
2 +"""Load and summarize prediction scorecards for reskill integration.
3 +
4 +Reads scorecard JSON files from data/metrics/{topic}/scorecards/ and produces
5 +a markdown summary suitable for injection into reskill prompts.
6 +"""
7 +
8 +from __future__ import annotations
9 +
10 +import json
11 +from pathlib import Path
12 +from typing import Any
13 +
14 +from scripts.topic_paths import metrics_dir
15 +
16 +DEFAULT_SCORECARD_COUNT = 4
17 +
18 +
19 +def scorecard_dir(topic_id: str | None = None) -> Path:
20 + """Return the scorecards directory for a given topic."""
21 + return metrics_dir(topic_id) / "scorecards"
22 +
23 +
24 +def load_scorecards(topic_id: str | None = None, count: int = DEFAULT_SCORECARD_COUNT) -> list[dict[str, Any]]:
25 + """Load the most recent N scorecards for a topic, sorted oldest-first."""
26 + directory = scorecard_dir(topic_id)
27 + if not directory.exists():
28 + return []
29 + files = sorted(directory.glob("*-scorecard.json"))
30 + if count > 0:
31 + files = files[-count:]
32 + cards: list[dict[str, Any]] = []
33 + for path in files:
34 + try:
35 + with open(path, encoding="utf-8") as f:
36 + cards.append(json.load(f))
37 + except (json.JSONDecodeError, OSError):
38 + continue
39 + return cards
40 +
41 +
42 +def _aggregate_stats(cards: list[dict[str, Any]]) -> tuple[int, int, int, dict[str, dict[str, int]]]:
43 + """Aggregate totals across multiple scorecards.
44 +
45 + Returns (total_validated, total_correct, total_incorrect, by_type).
46 + """
47 + total_validated = 0
48 + total_correct = 0
49 + total_incorrect = 0
50 + by_type: dict[str, dict[str, int]] = {}
51 +
52 + for card in cards:
53 + total_validated += card.get("validated", 0)
54 + total_correct += card.get("correct", 0)
55 + total_incorrect += card.get("incorrect", 0)
56 + for pred_type, stats in card.get("by_type", {}).items():
57 + if pred_type not in by_type:
58 + by_type[pred_type] = {"total": 0, "correct": 0}
59 + by_type[pred_type]["total"] += stats.get("total", 0)
60 + by_type[pred_type]["correct"] += stats.get("correct", 0)
61 +
62 + return total_validated, total_correct, total_incorrect, by_type
63 +
64 +
65 +def _format_by_type_analysis(by_type: dict[str, dict[str, int]]) -> list[str]:
66 + """Produce per-type accuracy lines for the summary."""
67 + lines: list[str] = []
68 + for pred_type, stats in sorted(by_type.items()):
69 + total = stats["total"]
70 + correct = stats["correct"]
71 + if total == 0:
72 + continue
73 + accuracy = correct / total
74 + pct = int(round(accuracy * 100))
75 + lines.append(f"- \"{pred_type}\" predictions: {pct}% accurate ({correct}/{total})")
76 + return lines
77 +
78 +
79 +def _format_recommendations(by_type: dict[str, dict[str, int]], overall_accuracy: float) -> list[str]:
80 + """Generate adjustment recommendations based on type performance."""
81 + recs: list[str] = []
82 + for pred_type, stats in sorted(by_type.items()):
83 + total = stats["total"]
84 + correct = stats["correct"]
85 + if total == 0:
86 + continue
87 + accuracy = correct / total
88 + if accuracy < 0.5:
89 + recs.append(
90 + f"- \"{pred_type}\" predictions are underperforming ({int(round(accuracy * 100))}%) "
91 + f"— raise confidence threshold or require additional signals"
92 + )
93 + elif accuracy >= 0.8:
94 + recs.append(
95 + f"- \"{pred_type}\" predictions are strong ({int(round(accuracy * 100))}%) "
96 + f"— current heuristics are reliable"
97 + )
98 + if not recs:
99 + if overall_accuracy < 0.6:
100 + recs.append("- Overall accuracy is low — review signal weighting across all prediction types")
101 + else:
102 + recs.append("- No specific type-level adjustments needed at this time")
103 + return recs
104 +
105 +
106 +def format_scorecard_summary(cards: list[dict[str, Any]]) -> str:
107 + """Format loaded scorecards into a markdown summary section.
108 +
109 + Returns empty string if cards is empty.
110 + """
111 + if not cards:
112 + return ""
113 +
114 + total_validated, total_correct, total_incorrect, by_type = _aggregate_stats(cards)
115 +
116 + if total_validated == 0:
117 + return ""
118 +
119 + overall_accuracy = total_correct / total_validated if total_validated else 0.0
120 + pct = int(round(overall_accuracy * 100))
121 + weeks = len(cards)
122 + week_range = f"last {weeks} week{'s' if weeks != 1 else ''}"
123 +
124 + lines: list[str] = []
125 + lines.append(f"## Prediction Performance ({week_range})")
126 + lines.append(f"Overall accuracy: {pct}% ({total_correct}/{total_validated} correct)")
127 + lines.append("")
128 + lines.append("### Per-Type Accuracy:")
129 + lines.extend(_format_by_type_analysis(by_type))
130 + lines.append("")
131 + lines.append("### Recommended Adjustments:")
132 + lines.extend(_format_recommendations(by_type, overall_accuracy))
133 +
134 + return "\n".join(lines)
135 +
136 +
137 +def render_scorecard_section(topic_id: str | None = None, count: int = DEFAULT_SCORECARD_COUNT) -> str:
138 + """Load scorecards and return formatted summary, or empty string if none exist."""
139 + cards = load_scorecards(topic_id, count)
140 + return format_scorecard_summary(cards)
scripts/reskill.py
+12
@@ -15,6 +15,7 @@ if str(ROOT) not in sys.path:
15 sys.path.insert(0, str(ROOT))
16
17 from scripts import track_quality
18 +from scripts.load_scorecard import render_scorecard_section
19
20 DEFAULT_PROMPT_TEMPLATE = ROOT / "prompts" / "reskill.md"
21 DEFAULT_ANALYZED_DIR = ROOT / "data" / "analyzed"
@@ -66,6 +67,9 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
67 help="Path to write the reskill report. Defaults to .squad/reskill/YYYY-WNN.md.",
68 )
69 parser.add_argument("--limit", type=int, default=5, help="Maximum number of analyzed summaries to include.")
70 + parser.add_argument("--scorecard", action="store_true", help="Include prediction scorecard data in the reskill prompt.")
71 + parser.add_argument("--scorecard-count", type=int, default=4, help="Number of recent scorecards to include (default: 4).")
72 + parser.add_argument("--topic", default=None, help="Topic ID for scorecard resolution.")
73 parser.add_argument(
74 "--print-prompt",
75 action="store_true",
@@ -185,6 +189,7 @@ def render_prompt(
189 wisdom_file: Path,
190 skills_dir: Path,
191 limit: int,
192 + scorecard_section: str = "",
193 ) -> str:
194 prompt = prompt_template_path.read_text(encoding="utf-8")
195 replacements = {
@@ -195,6 +200,7 @@ def render_prompt(
200 "{{QUALITY_TREND}}": track_quality.build_quality_report(analyzed_dir).strip(),
201 "{{RECENT_ANALYSES}}": render_recent_analyses(analyzed_dir, limit),
202 "{{SNAPSHOT_CONTEXT}}": render_snapshot_context(analyzed_dir, snapshots_dir, limit),
203 + "{{SCORECARD}}": scorecard_section,
204 }
205 for needle, value in replacements.items():
206 prompt = prompt.replace(needle, value)
@@ -269,6 +275,11 @@ def call_github_models(prompt: str) -> str:
275 def main(argv: list[str] | None = None) -> int:
276 args = parse_args(argv)
277 output_path = args.output or default_output_path(args.current_datetime)
278 +
279 + scorecard_section = ""
280 + if args.scorecard:
281 + scorecard_section = render_scorecard_section(args.topic, args.scorecard_count)
282 +
283 prompt = render_prompt(
284 prompt_template_path=args.prompt_template,
285 current_datetime=args.current_datetime,
@@ -278,6 +289,7 @@ def main(argv: list[str] | None = None) -> int:
289 wisdom_file=args.wisdom_file,
290 skills_dir=args.skills_dir,
291 limit=args.limit,
292 + scorecard_section=scorecard_section,
293 )
294
295 if args.print_prompt:
tests/test_end_to_end_topic.py new
+355
@@ -0,0 +1,355 @@
1 +#!/usr/bin/env python3
2 +"""End-to-end integration test for the SquadScope topic pipeline with ai-ml config.
3 +
4 +Exercises the full pipeline: validate → crawl (mocked) → score → quality gate →
5 +prediction ledger → content verification, using realistic AI/ML mock data.
6 +"""
7 +
8 +from __future__ import annotations
9 +
10 +import json
11 +import os
12 +import shutil
13 +import tempfile
14 +import unittest
15 +from pathlib import Path
16 +
17 +import yaml
18 +
19 +
20 +REPO_ROOT = Path(__file__).resolve().parent.parent
21 +
22 +MOCK_REPOS = [
23 + {
24 + "full_name": "org/pytorch-trainer",
25 + "language": "Python",
26 + "topics": ["machine-learning", "pytorch", "deep-learning"],
27 + "stars": 500,
28 + "stars_gained": 50,
29 + "forks": 80,
30 + "created_at": "2024-06-01T00:00:00Z",
31 + "description": "High-performance PyTorch training framework",
32 + },
33 + {
34 + "full_name": "org/llm-finetune",
35 + "language": "Python",
36 + "topics": ["llm", "transformers", "machine-learning"],
37 + "stars": 1200,
38 + "stars_gained": 200,
39 + "forks": 150,
40 + "created_at": "2024-03-15T00:00:00Z",
41 + "description": "Fine-tuning toolkit for large language models",
42 + },
43 + {
44 + "full_name": "org/neural-notebook",
45 + "language": "Jupyter Notebook",
46 + "topics": ["deep-learning", "neural-network"],
47 + "stars": 80,
48 + "stars_gained": 30,
49 + "forks": 12,
50 + "created_at": "2025-01-10T00:00:00Z",
51 + "description": "Interactive notebooks for neural network experiments",
52 + },
53 + {
54 + "full_name": "org/ai-image-gen",
55 + "language": "Python",
56 + "topics": ["artificial-intelligence", "deep-learning", "transformers"],
57 + "stars": 3000,
58 + "stars_gained": 400,
59 + "forks": 500,
60 + "created_at": "2023-11-20T00:00:00Z",
61 + "description": "State-of-the-art image generation with diffusion models",
62 + },
63 + {
64 + "full_name": "org/ml-pipeline",
65 + "language": "Python",
66 + "topics": ["machine-learning", "mlops"],
67 + "stars": 250,
68 + "stars_gained": 35,
69 + "forks": 40,
70 + "created_at": "2024-09-01T00:00:00Z",
71 + "description": "End-to-end ML pipeline orchestration",
72 + },
73 + {
74 + "full_name": "org/transformer-serving",
75 + "language": "Python",
76 + "topics": ["transformers", "llm", "inference"],
77 + "stars": 900,
78 + "stars_gained": 120,
79 + "forks": 95,
80 + "created_at": "2024-05-01T00:00:00Z",
81 + "description": "Scalable transformer model serving",
82 + },
83 + {
84 + "full_name": "org/rust-cli-tool",
85 + "language": "Rust",
86 + "topics": ["cli-tool", "developer-tools"],
87 + "stars": 300,
88 + "stars_gained": 15,
89 + "forks": 20,
90 + "created_at": "2024-07-01T00:00:00Z",
91 + "description": "Fast CLI utility written in Rust",
92 + },
93 + {
94 + "full_name": "org/web-dashboard",
95 + "language": "JavaScript",
96 + "topics": ["web", "dashboard", "react"],
97 + "stars": 400,
98 + "stars_gained": 25,
99 + "forks": 50,
100 + "created_at": "2024-04-01T00:00:00Z",
101 + "description": "Modern web dashboard framework",
102 + },
103 + {
104 + "full_name": "org/tiny-nn",
105 + "language": "Python",
106 + "topics": ["neural-network", "deep-learning"],
107 + "stars": 150,
108 + "stars_gained": 45,
109 + "forks": 25,
110 + "created_at": "2025-02-01T00:00:00Z",
111 + "description": "Minimal neural network library for education",
112 + },
113 + {
114 + "full_name": "org/data-viz",
115 + "language": "JavaScript",
116 + "topics": ["visualization", "charts"],
117 + "stars": 600,
118 + "stars_gained": 20,
119 + "forks": 70,
120 + "created_at": "2023-06-01T00:00:00Z",
121 + "description": "Data visualization library",
122 + },
123 +]
124 +
125 +# Raw data format expected by prediction_ledger
126 +MOCK_RAW_DATA = {
127 + "week": "2025-W25",
128 + "new_repos": MOCK_REPOS[:5],
129 + "trending_repos": MOCK_REPOS[5:],
130 +}
131 +
132 +MOCK_SUMMARY_CONTENT = """---
133 +topic: ai-ml
134 +week: 2025-W25
135 +repos_scored: 8
136 +---
137 +
138 +# AI & ML Weekly Digest — 2025-W25
139 +
140 +## Highlights
141 +
142 +- [org/pytorch-trainer](https://github.com/org/pytorch-trainer) — High-performance training
143 +- [org/llm-finetune](https://github.com/org/llm-finetune) — LLM fine-tuning toolkit
144 +- [org/ai-image-gen](https://github.com/org/ai-image-gen) — Diffusion image generation
145 +- [org/transformer-serving](https://github.com/org/transformer-serving) — Model serving at scale
146 +"""
147 +
148 +
149 +class TestEndToEndTopicPipeline(unittest.TestCase):
150 + """Full pipeline integration test with ai-ml topic config."""
151 +
152 + def setUp(self):
153 + """Create temp directory structure with ai-ml config and mock data."""
154 + self.temp_dir = tempfile.TemporaryDirectory()
155 + self.work_dir = Path(self.temp_dir.name)
156 +
157 + # Copy the real ai-ml config
158 + src_config = REPO_ROOT / "squadscope.topic.yml"
159 + self.config_path = self.work_dir / "squadscope.topic.yml"
160 + shutil.copy(src_config, self.config_path)
161 +
162 + # Create data directories
163 + self.raw_path = self.work_dir / "data" / "raw" / "ai-ml"
164 + self.analyzed_path = self.work_dir / "data" / "analyzed" / "ai-ml"
165 + self.metrics_path = self.work_dir / "data" / "metrics" / "ai-ml"
166 + for d in (self.raw_path, self.analyzed_path, self.metrics_path):
167 + d.mkdir(parents=True, exist_ok=True)
168 +
169 + # Write mock raw crawl data (list format for score_repos)
170 + self.raw_json_path = self.raw_path / "2025-W25.json"
171 + self.raw_json_path.write_text(json.dumps(MOCK_REPOS, indent=2), encoding="utf-8")
172 +
173 + # Write raw data in dict format for prediction_ledger
174 + self.raw_dict_path = self.raw_path / "2025-W25-full.json"
175 + self.raw_dict_path.write_text(json.dumps(MOCK_RAW_DATA, indent=2), encoding="utf-8")
176 +
177 + # Write mock summary for prediction ledger
178 + self.summary_path = self.analyzed_path / "2025-W25-summary.md"
179 + self.summary_path.write_text(MOCK_SUMMARY_CONTENT, encoding="utf-8")
180 +
181 + # Change to work dir so relative paths in scripts work
182 + self._orig_cwd = os.getcwd()
183 + os.chdir(self.work_dir)
184 +
185 + def tearDown(self):
186 + os.chdir(self._orig_cwd)
187 + self.temp_dir.cleanup()
188 +
189 + def test_full_pipeline_ai_ml(self):
190 + """Run the full pipeline: validate → score → quality gate → predictions."""
191 + import sys
192 + sys.path.insert(0, str(REPO_ROOT))
193 +
194 + from scripts.validate_topic_config import validate_file
195 + from scripts.score_repos import score_repos, load_config, get_scoring_config
196 + from scripts.quality_gate import check_quality, get_quality_config, write_metric, week_slug
197 + from scripts.prediction_ledger import generate_predictions, append_predictions
198 +
199 + # --- Stage 1: Validate config ---
200 + config_model = validate_file(str(self.config_path))
201 + self.assertEqual(config_model.topic.id, "ai-ml")
202 + self.assertEqual(config_model.topic.name, "AI & Machine Learning")
203 + self.assertGreater(len(config_model.queries.primary), 0)
204 + self.assertIn("Python", config_model.scoring.language_boost)
205 +
206 + # --- Stage 2: Mock crawl data already created in setUp ---
207 + raw_repos = json.loads(self.raw_json_path.read_text(encoding="utf-8"))
208 + self.assertEqual(len(raw_repos), 10)
209 +
210 + # --- Stage 3: Score repos ---
211 + config = load_config(self.config_path)
212 + scoring_config = get_scoring_config(config)
213 + scored = score_repos(raw_repos, scoring_config)
214 +
215 + # Verify scoring filters and ranks correctly
216 + self.assertGreater(len(scored), 0)
217 + self.assertLessEqual(len(scored), len(raw_repos))
218 +
219 + # All scored repos should meet minimum relevance threshold
220 + min_score = scoring_config["min_relevance_score"]
221 + for repo in scored:
222 + self.assertGreaterEqual(repo["relevance_score"], min_score)
223 + self.assertIn("relevance_score", repo)
224 +
225 + # Verify sorted descending
226 + scores = [r["relevance_score"] for r in scored]
227 + self.assertEqual(scores, sorted(scores, reverse=True))
228 +
229 + # AI/ML repos should score higher than non-AI repos
230 + ai_repos = [r for r in scored if r["full_name"] in (
231 + "org/pytorch-trainer", "org/llm-finetune", "org/ai-image-gen"
232 + )]
233 + non_ai_repos = [r for r in scored if r["full_name"] in (
234 + "org/rust-cli-tool", "org/web-dashboard", "org/data-viz"
235 + )]
236 + if ai_repos and non_ai_repos:
237 + max_non_ai = max(r["relevance_score"] for r in non_ai_repos)
238 + min_ai = min(r["relevance_score"] for r in ai_repos)
239 + self.assertGreater(min_ai, max_non_ai,
240 + "AI/ML repos should outscore non-AI repos")
241 +
242 + # Write scored output for quality gate
243 + scored_path = self.work_dir / "scored.json"
244 + scored_path.write_text(json.dumps(scored, indent=2), encoding="utf-8")
245 +
246 + # --- Stage 4: Quality gate ---
247 + quality_config = get_quality_config(config)
248 + scoring_cfg = config.get("scoring", {})
249 + metric = check_quality(scored, quality_config, scoring_cfg)
250 +
251 + self.assertIn("repos_scored", metric)
252 + self.assertIn("repos_passing", metric)
253 + self.assertIn("status", metric)
254 + self.assertIn("warnings", metric)
255 + self.assertGreater(metric["repos_scored"], 0)
256 +
257 + # Write metric file
258 + week = "2025-W25"
259 + metric_path = write_metric("ai-ml", metric, week)
260 + self.assertTrue(metric_path.exists())
261 + metric_data = json.loads(metric_path.read_text(encoding="utf-8"))
262 + self.assertEqual(metric_data["topic"], "ai-ml")
263 + self.assertEqual(metric_data["week"], week)
264 +
265 + # --- Stage 5: Prediction ledger ---
266 + summary_content = self.summary_path.read_text(encoding="utf-8")
267 + raw_data = json.loads(self.raw_dict_path.read_text(encoding="utf-8"))
268 + predictions = generate_predictions(summary_content, raw_data, week)
269 +
270 + self.assertGreaterEqual(len(predictions), 1)
271 + self.assertLessEqual(len(predictions), 5)
272 +
273 + for pred in predictions:
274 + self.assertIn("week", pred)
275 + self.assertIn("repo", pred)
276 + self.assertIn("prediction", pred)
277 + self.assertIn("confidence", pred)
278 + self.assertIn("reason", pred)
279 + self.assertEqual(pred["week"], week)
280 + self.assertGreater(pred["confidence"], 0)
281 + self.assertIn(pred["prediction"], [
282 + "rising_star", "emerging_topic", "momentum_shift",
283 + "breakout_candidate", "declining_signal",
284 + ])
285 +
286 + # Write predictions to ledger
287 + predictions_path = self.metrics_path / "predictions.jsonl"
288 + append_predictions(predictions, predictions_path)
289 + self.assertTrue(predictions_path.exists())
290 +
291 + # Verify JSONL format
292 + lines = predictions_path.read_text(encoding="utf-8").strip().split("\n")
293 + self.assertEqual(len(lines), len(predictions))
294 + for line in lines:
295 + entry = json.loads(line)
296 + self.assertIn("repo", entry)
297 + self.assertIn("prediction", entry)
298 +
299 + # --- Stage 6: Content generation verification ---
300 + # Verify scored data is suitable for content generation
301 + top_repos = scored[:5]
302 + for repo in top_repos:
303 + self.assertIn("full_name", repo)
304 + self.assertIn("description", repo)
305 + self.assertIn("relevance_score", repo)
306 + # Verify topic-specific content (not generic)
307 + topics = repo.get("topics", [])
308 + ai_topics = {"machine-learning", "deep-learning", "artificial-intelligence",
309 + "neural-network", "llm", "transformers"}
310 + has_ai_topic = bool(set(t.lower() for t in topics) & ai_topics)
311 + self.assertTrue(has_ai_topic,
312 + f"Top repo {repo['full_name']} should have AI/ML topics")
313 +
314 + def test_config_validation_rejects_invalid(self):
315 + """Ensure validate rejects malformed configs."""
316 + import sys
317 + sys.path.insert(0, str(REPO_ROOT))
318 + from scripts.validate_topic_config import validate_file
319 +
320 + # Write an invalid config (missing required fields)
321 + bad_config = self.work_dir / "bad.yml"
322 + bad_config.write_text("topic:\n id: INVALID ID WITH SPACES\n", encoding="utf-8")
323 +
324 + with self.assertRaises(Exception):
325 + validate_file(str(bad_config))
326 +
327 + def test_scoring_respects_language_boost(self):
328 + """Verify Python repos get a language boost over unlisted languages."""
329 + import sys
330 + sys.path.insert(0, str(REPO_ROOT))
331 + from scripts.score_repos import compute_relevance_score, load_config, get_scoring_config
332 +
333 + config = load_config(self.config_path)
334 + scoring_config = get_scoring_config(config)
335 +
336 + base_repo = {
337 + "full_name": "test/repo",
338 + "stars": 500,
339 + "stars_gained": 50,
340 + "topics": ["machine-learning"],
341 + "created_at": "2024-06-01T00:00:00Z",
342 + }
343 +
344 + python_repo = {**base_repo, "language": "Python"}
345 + rust_repo = {**base_repo, "language": "Rust"}
346 +
347 + python_score = compute_relevance_score(python_repo, scoring_config)
348 + rust_score = compute_relevance_score(rust_repo, scoring_config)
349 +
350 + self.assertGreater(python_score, rust_score,
351 + "Python should score higher with language_boost configured")
352 +
353 +
354 +if __name__ == "__main__":
355 + unittest.main()
tests/test_load_scorecard.py new
+150
@@ -0,0 +1,150 @@
1 +"""Tests for scripts/load_scorecard.py"""
2 +
3 +from __future__ import annotations
4 +
5 +import json
6 +from pathlib import Path
7 +
8 +import pytest
9 +
10 +from scripts.load_scorecard import (
11 + format_scorecard_summary,
12 + load_scorecards,
13 + render_scorecard_section,
14 + scorecard_dir,
15 +)
16 +
17 +
18 +def _make_scorecard(week: str, topic: str = "ai-ml", validated: int = 5, correct: int = 3, incorrect: int = 2, by_type: dict | None = None) -> dict:
19 + return {
20 + "week": week,
21 + "topic": topic,
22 + "total_predictions": validated + 2,
23 + "validated": validated,
24 + "correct": correct,
25 + "incorrect": incorrect,
26 + "accuracy": correct / validated if validated else 0,
27 + "by_type": by_type or {"rising_star": {"total": 3, "correct": 2}, "declining_signal": {"total": 2, "correct": 1}},
28 + "details": [],
29 + }
30 +
31 +
32 +@pytest.fixture
33 +def scorecards_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
34 + """Create a fake scorecards directory and patch metrics_dir."""
35 + topic = "ai-ml"
36 + sc_dir = tmp_path / "data" / "metrics" / topic / "scorecards"
37 + sc_dir.mkdir(parents=True)
38 +
39 + monkeypatch.setattr("scripts.load_scorecard.metrics_dir", lambda topic_id=None: tmp_path / "data" / "metrics" / (topic_id or "general"))
40 +
41 + return sc_dir
42 +
43 +
44 +class TestScorecardDir:
45 + def test_returns_scorecards_subdir(self):
46 + path = scorecard_dir("ai-ml")
47 + assert path.name == "scorecards"
48 + assert "ai-ml" in str(path)
49 +
50 + def test_general_topic(self):
51 + path = scorecard_dir(None)
52 + assert "scorecards" in str(path)
53 +
54 +
55 +class TestLoadScorecards:
56 + def test_empty_when_no_dir(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
57 + monkeypatch.setattr("scripts.load_scorecard.metrics_dir", lambda topic_id=None: tmp_path / "nonexistent")
58 + result = load_scorecards("ai-ml")
59 + assert result == []
60 +
61 + def test_loads_recent_scorecards(self, scorecards_dir: Path):
62 + for i, week in enumerate(["2026-W18", "2026-W19", "2026-W20", "2026-W21", "2026-W22"]):
63 + card = _make_scorecard(week, correct=i + 1, validated=5)
64 + (scorecards_dir / f"{week}-scorecard.json").write_text(json.dumps(card))
65 +
66 + cards = load_scorecards("ai-ml", count=4)
67 + assert len(cards) == 4
68 + assert cards[0]["week"] == "2026-W19"
69 + assert cards[-1]["week"] == "2026-W22"
70 +
71 + def test_loads_all_when_fewer_than_count(self, scorecards_dir: Path):
72 + card = _make_scorecard("2026-W21")
73 + (scorecards_dir / "2026-W21-scorecard.json").write_text(json.dumps(card))
74 +
75 + cards = load_scorecards("ai-ml", count=4)
76 + assert len(cards) == 1
77 +
78 + def test_skips_invalid_json(self, scorecards_dir: Path):
79 + (scorecards_dir / "2026-W20-scorecard.json").write_text("not json")
80 + card = _make_scorecard("2026-W21")
81 + (scorecards_dir / "2026-W21-scorecard.json").write_text(json.dumps(card))
82 +
83 + cards = load_scorecards("ai-ml", count=4)
84 + assert len(cards) == 1
85 + assert cards[0]["week"] == "2026-W21"
86 +
87 +
88 +class TestFormatScorecardSummary:
89 + def test_empty_cards_returns_empty(self):
90 + assert format_scorecard_summary([]) == ""
91 +
92 + def test_zero_validated_returns_empty(self):
93 + card = _make_scorecard("2026-W21", validated=0, correct=0, incorrect=0)
94 + assert format_scorecard_summary([card]) == ""
95 +
96 + def test_single_card_summary(self):
97 + card = _make_scorecard("2026-W21", validated=5, correct=3, incorrect=2)
98 + result = format_scorecard_summary([card])
99 +
100 + assert "## Prediction Performance (last 1 week)" in result
101 + assert "Overall accuracy: 60% (3/5 correct)" in result
102 + assert "rising_star" in result
103 + assert "declining_signal" in result
104 +
105 + def test_multiple_cards_aggregate(self):
106 + cards = [
107 + _make_scorecard("2026-W20", validated=5, correct=4, incorrect=1,
108 + by_type={"rising_star": {"total": 3, "correct": 2}, "breakout": {"total": 2, "correct": 2}}),
109 + _make_scorecard("2026-W21", validated=5, correct=3, incorrect=2,
110 + by_type={"rising_star": {"total": 3, "correct": 1}, "breakout": {"total": 2, "correct": 2}}),
111 + ]
112 + result = format_scorecard_summary(cards)
113 +
114 + assert "last 2 weeks" in result
115 + assert "70% (7/10 correct)" in result
116 + # rising_star: 3/6 = 50%
117 + assert "\"rising_star\" predictions: 50%" in result
118 + # breakout: 4/4 = 100%
119 + assert "\"breakout\" predictions: 100%" in result
120 +
121 + def test_recommendations_for_low_accuracy(self):
122 + cards = [
123 + _make_scorecard("2026-W21", validated=10, correct=3, incorrect=7,
124 + by_type={"rising_star": {"total": 10, "correct": 3}}),
125 + ]
126 + result = format_scorecard_summary(cards)
127 + assert "raise confidence threshold" in result
128 +
129 + def test_recommendations_for_high_accuracy(self):
130 + cards = [
131 + _make_scorecard("2026-W21", validated=10, correct=9, incorrect=1,
132 + by_type={"declining_signal": {"total": 10, "correct": 9}}),
133 + ]
134 + result = format_scorecard_summary(cards)
135 + assert "reliable" in result
136 +
137 +
138 +class TestRenderScorecardSection:
139 + def test_returns_empty_when_no_scorecards(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
140 + monkeypatch.setattr("scripts.load_scorecard.metrics_dir", lambda topic_id=None: tmp_path / "nonexistent")
141 + result = render_scorecard_section("ai-ml")
142 + assert result == ""
143 +
144 + def test_returns_formatted_summary(self, scorecards_dir: Path):
145 + card = _make_scorecard("2026-W21", validated=5, correct=4, incorrect=1)
146 + (scorecards_dir / "2026-W21-scorecard.json").write_text(json.dumps(card))
147 +
148 + result = render_scorecard_section("ai-ml")
149 + assert "Prediction Performance" in result
150 + assert "80%" in result