@cryptotaxi247 / SquadScope
Code Commits Issues Pulls Sessions CI/CD
main
main
py 196 lines 6.28 KB
Raw
1 """Tests for scripts/load_scorecard.py"""
2
3 from __future__ import annotations
4
5 import json
6 from pathlib import Path
7
8 import pytest
9
10 from scripts.load_scorecard import (
11 format_scorecard_summary,
12 load_scorecards,
13 render_scorecard_section,
14 scorecard_dir,
15 )
16
17
18 def _make_scorecard(
19 week: str,
20 topic: str = "ai-ml",
21 validated: int = 5,
22 correct: int = 3,
23 incorrect: int = 2,
24 by_type: dict | None = None,
25 ) -> dict:
26 return {
27 "week": week,
28 "topic": topic,
29 "total_predictions": validated + 2,
30 "validated": validated,
31 "correct": correct,
32 "incorrect": incorrect,
33 "accuracy": correct / validated if validated else 0,
34 "by_type": by_type
35 or {
36 "rising_star": {"total": 3, "correct": 2},
37 "declining_signal": {"total": 2, "correct": 1},
38 },
39 "details": [],
40 }
41
42
43 @pytest.fixture
44 def scorecards_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
45 """Create a fake scorecards directory and patch metrics_dir."""
46 topic = "ai-ml"
47 sc_dir = tmp_path / "data" / "metrics" / topic / "scorecards"
48 sc_dir.mkdir(parents=True)
49
50 monkeypatch.setattr(
51 "scripts.load_scorecard.metrics_dir",
52 lambda topic_id=None: tmp_path / "data" / "metrics" / (topic_id or "general"),
53 )
54
55 return sc_dir
56
57
58 class TestScorecardDir:
59 def test_returns_scorecards_subdir(self):
60 path = scorecard_dir("ai-ml")
61 assert path.name == "scorecards"
62 assert "ai-ml" in str(path)
63
64 def test_general_topic(self):
65 path = scorecard_dir(None)
66 assert "scorecards" in str(path)
67
68
69 class TestLoadScorecards:
70 def test_empty_when_no_dir(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
71 monkeypatch.setattr(
72 "scripts.load_scorecard.metrics_dir", lambda topic_id=None: tmp_path / "nonexistent"
73 )
74 result = load_scorecards("ai-ml")
75 assert result == []
76
77 def test_loads_recent_scorecards(self, scorecards_dir: Path):
78 for i, week in enumerate(["2026-W18", "2026-W19", "2026-W20", "2026-W21", "2026-W22"]):
79 card = _make_scorecard(week, correct=i + 1, validated=5)
80 (scorecards_dir / f"{week}-scorecard.json").write_text(json.dumps(card))
81
82 cards = load_scorecards("ai-ml", count=4)
83 assert len(cards) == 4
84 assert cards[0]["week"] == "2026-W19"
85 assert cards[-1]["week"] == "2026-W22"
86
87 def test_loads_all_when_fewer_than_count(self, scorecards_dir: Path):
88 card = _make_scorecard("2026-W21")
89 (scorecards_dir / "2026-W21-scorecard.json").write_text(json.dumps(card))
90
91 cards = load_scorecards("ai-ml", count=4)
92 assert len(cards) == 1
93
94 def test_skips_invalid_json(self, scorecards_dir: Path):
95 (scorecards_dir / "2026-W20-scorecard.json").write_text("not json")
96 card = _make_scorecard("2026-W21")
97 (scorecards_dir / "2026-W21-scorecard.json").write_text(json.dumps(card))
98
99 cards = load_scorecards("ai-ml", count=4)
100 assert len(cards) == 1
101 assert cards[0]["week"] == "2026-W21"
102
103
104 class TestFormatScorecardSummary:
105 def test_empty_cards_returns_empty(self):
106 assert format_scorecard_summary([]) == ""
107
108 def test_zero_validated_returns_empty(self):
109 card = _make_scorecard("2026-W21", validated=0, correct=0, incorrect=0)
110 assert format_scorecard_summary([card]) == ""
111
112 def test_single_card_summary(self):
113 card = _make_scorecard("2026-W21", validated=5, correct=3, incorrect=2)
114 result = format_scorecard_summary([card])
115
116 assert "## Prediction Performance (last 1 week)" in result
117 assert "Overall accuracy: 60% (3/5 correct)" in result
118 assert "rising_star" in result
119 assert "declining_signal" in result
120
121 def test_multiple_cards_aggregate(self):
122 cards = [
123 _make_scorecard(
124 "2026-W20",
125 validated=5,
126 correct=4,
127 incorrect=1,
128 by_type={
129 "rising_star": {"total": 3, "correct": 2},
130 "breakout": {"total": 2, "correct": 2},
131 },
132 ),
133 _make_scorecard(
134 "2026-W21",
135 validated=5,
136 correct=3,
137 incorrect=2,
138 by_type={
139 "rising_star": {"total": 3, "correct": 1},
140 "breakout": {"total": 2, "correct": 2},
141 },
142 ),
143 ]
144 result = format_scorecard_summary(cards)
145
146 assert "last 2 weeks" in result
147 assert "70% (7/10 correct)" in result
148 # rising_star: 3/6 = 50%
149 assert '"rising_star" predictions: 50%' in result
150 # breakout: 4/4 = 100%
151 assert '"breakout" predictions: 100%' in result
152
153 def test_recommendations_for_low_accuracy(self):
154 cards = [
155 _make_scorecard(
156 "2026-W21",
157 validated=10,
158 correct=3,
159 incorrect=7,
160 by_type={"rising_star": {"total": 10, "correct": 3}},
161 ),
162 ]
163 result = format_scorecard_summary(cards)
164 assert "raise confidence threshold" in result
165
166 def test_recommendations_for_high_accuracy(self):
167 cards = [
168 _make_scorecard(
169 "2026-W21",
170 validated=10,
171 correct=9,
172 incorrect=1,
173 by_type={"declining_signal": {"total": 10, "correct": 9}},
174 ),
175 ]
176 result = format_scorecard_summary(cards)
177 assert "reliable" in result
178
179
180 class TestRenderScorecardSection:
181 def test_returns_empty_when_no_scorecards(
182 self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
183 ):
184 monkeypatch.setattr(
185 "scripts.load_scorecard.metrics_dir", lambda topic_id=None: tmp_path / "nonexistent"
186 )
187 result = render_scorecard_section("ai-ml")
188 assert result == ""
189
190 def test_returns_formatted_summary(self, scorecards_dir: Path):
191 card = _make_scorecard("2026-W21", validated=5, correct=4, incorrect=1)
192 (scorecards_dir / "2026-W21-scorecard.json").write_text(json.dumps(card))
193
194 result = render_scorecard_section("ai-ml")
195 assert "Prediction Performance" in result
196 assert "80%" in result