test: validate ai-ml and rust topic configs (#70, #71) (#101)
Integration tests that validate schema and scoring pipeline end-to-end for both ai-ml and rust topic configurations using mock repo data. Tests cover schema validation, scoring reasonableness, language boost behavior, min_repos_per_week achievability, and cross-topic isolation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 19, 2026 at 15:58 UTC
934571da78f615922d667fe7ff8a73965878f476
1 file changed
+273
tests/test_topic_config_validation.py
new
+273
@@ -0,0 +1,273 @@
1
+"""Integration tests: validate ai-ml and rust topic configs end-to-end.
2
+
3
+Verifies schema validation passes and scoring pipeline produces reasonable
4
+results with mock repo data for both topics, plus cross-topic isolation.
5
+"""
6
+
7
+from __future__ import annotations
8
+
9
+from datetime import UTC, datetime, timedelta
10
+
11
+import pytest
12
+
13
+from scripts.score_repos import compute_relevance_score, get_scoring_config, load_config, score_repos
14
+from scripts.validate_topic_config import validate_file
15
+
16
+
17
+# --- Helpers ---
18
+
19
+
20
+def _make_repo(
21
+ name: str,
22
+ language: str,
23
+ stars: int,
24
+ stars_gained: int,
25
+ topics: list[str],
26
+ age_days: int = 30,
27
+) -> dict:
28
+ """Create a mock repo dict with given characteristics."""
29
+ return {
30
+ "name": name,
31
+ "owner": "mock-org",
32
+ "full_name": f"mock-org/{name}",
33
+ "description": f"Mock repo: {name}",
34
+ "language": language,
35
+ "stars": stars,
36
+ "forks": stars // 10,
37
+ "created_at": (datetime.now(UTC) - timedelta(days=age_days)).isoformat(),
38
+ "topics": topics,
39
+ "license": "MIT",
40
+ "url": f"https://github.com/mock-org/{name}",
41
+ "stars_gained": stars_gained,
42
+ }
43
+
44
+
45
+# --- AI/ML Config Fixtures ---
46
+
47
+
48
+@pytest.fixture
49
+def aiml_root_config():
50
+ return validate_file("squadscope.topic.yml")
51
+
52
+
53
+@pytest.fixture
54
+def aiml_example_config():
55
+ return validate_file("examples/topics/ai-ml.yml")
56
+
57
+
58
+@pytest.fixture
59
+def aiml_scoring_config():
60
+ config = load_config("squadscope.topic.yml")
61
+ return get_scoring_config(config)
62
+
63
+
64
+@pytest.fixture
65
+def rust_config():
66
+ return validate_file("examples/topics/rust.yml")
67
+
68
+
69
+@pytest.fixture
70
+def rust_scoring_config():
71
+ config = load_config("examples/topics/rust.yml")
72
+ return get_scoring_config(config)
73
+
74
+
75
+@pytest.fixture
76
+def aiml_repos():
77
+ """Sample repos matching AI/ML profile."""
78
+ return [
79
+ _make_repo("transformer-lib", "Python", 500, 120, ["machine-learning", "transformers", "deep-learning"]),
80
+ _make_repo("llm-toolkit", "Python", 1200, 300, ["llm", "artificial-intelligence", "python"]),
81
+ _make_repo("ml-starter", "Jupyter Notebook", 150, 40, ["machine-learning", "neural-network"]),
82
+ _make_repo("data-pipeline", "Python", 80, 20, ["machine-learning"], age_days=60),
83
+ _make_repo("ai-research", "Python", 3000, 500, ["deep-learning", "llm", "transformers"], age_days=10),
84
+ _make_repo("small-ml", "Python", 50, 15, ["machine-learning"], age_days=90),
85
+ _make_repo("mid-ml", "Python", 200, 50, ["deep-learning", "neural-network"], age_days=45),
86
+ ]
87
+
88
+
89
+@pytest.fixture
90
+def rust_repos():
91
+ """Sample repos matching Rust profile."""
92
+ return [
93
+ _make_repo("fast-cli", "Rust", 400, 80, ["rust", "cli", "systems-programming"]),
94
+ _make_repo("async-runtime", "Rust", 2000, 200, ["rust", "async-rust", "cargo"]),
95
+ _make_repo("wasm-toolkit", "Rust", 600, 100, ["rust", "wasm", "cargo"]),
96
+ _make_repo("rust-game", "Rust", 150, 40, ["rust", "systems-programming"]),
97
+ _make_repo("tiny-crate", "Rust", 50, 20, ["rust", "cargo"], age_days=60),
98
+ ]
99
+
100
+
101
+# =============================================================================
102
+# 1. AI/ML Schema Validation
103
+# =============================================================================
104
+
105
+
106
+class TestAimlSchemaValidation:
107
+ """Issue #70: Validate ai-ml configs pass schema validation."""
108
+
109
+ def test_root_config_passes_validation(self, aiml_root_config):
110
+ assert aiml_root_config.topic.id == "ai-ml"
111
+ assert aiml_root_config.topic.name == "AI & Machine Learning"
112
+ assert len(aiml_root_config.queries.primary) >= 2
113
+
114
+ def test_example_config_passes_validation(self, aiml_example_config):
115
+ assert aiml_example_config.topic.id == "ai-ml"
116
+ assert aiml_example_config.scoring.min_stars == 20
117
+ assert aiml_example_config.scoring.min_relevance_score == 40
118
+
119
+ def test_root_config_scoring_section(self, aiml_root_config):
120
+ assert aiml_root_config.scoring.language_boost.get("Python") == 1.2
121
+ assert "machine-learning" in aiml_root_config.scoring.topic_relevance
122
+
123
+ def test_root_config_quality_section(self, aiml_root_config):
124
+ assert aiml_root_config.quality.min_repos_per_week == 5
125
+ assert aiml_root_config.quality.max_repos_per_week == 30
126
+
127
+ def test_root_config_learning_section(self, aiml_root_config):
128
+ assert "ai-ml" in aiml_root_config.learning.wisdom_file
129
+
130
+
131
+# =============================================================================
132
+# 2. AI/ML Scoring Pipeline
133
+# =============================================================================
134
+
135
+
136
+class TestAimlScoringPipeline:
137
+ """Issue #70: AI/ML repos get reasonable scores with mock data."""
138
+
139
+ def test_typical_aiml_repos_score_above_threshold(self, aiml_scoring_config, aiml_repos):
140
+ scored = score_repos(aiml_repos, aiml_scoring_config)
141
+ # All well-formed AI/ML repos should pass the min_relevance_score (40)
142
+ assert len(scored) >= 5
143
+
144
+ def test_high_quality_repo_scores_above_40(self, aiml_scoring_config):
145
+ repo = _make_repo("top-ml", "Python", 500, 100, ["machine-learning", "deep-learning", "llm"])
146
+ score = compute_relevance_score(repo, aiml_scoring_config)
147
+ assert score >= 40
148
+
149
+ def test_min_repos_per_week_achievable(self, aiml_scoring_config, aiml_repos):
150
+ """With typical data, at least min_repos_per_week pass threshold."""
151
+ min_required = 5 # from config quality.min_repos_per_week
152
+ scored = score_repos(aiml_repos, aiml_scoring_config)
153
+ assert len(scored) >= min_required
154
+
155
+ def test_python_language_boost_applies(self, aiml_scoring_config):
156
+ python_repo = _make_repo("py-ml", "Python", 200, 50, ["machine-learning"])
157
+ other_repo = _make_repo("go-ml", "Go", 200, 50, ["machine-learning"])
158
+
159
+ py_score = compute_relevance_score(python_repo, aiml_scoring_config)
160
+ go_score = compute_relevance_score(other_repo, aiml_scoring_config)
161
+ assert py_score > go_score
162
+
163
+ def test_topic_relevance_boosts_score(self, aiml_scoring_config):
164
+ relevant = _make_repo("relevant", "Python", 200, 50, ["machine-learning", "deep-learning", "llm"])
165
+ irrelevant = _make_repo("irrelevant", "Python", 200, 50, ["cooking", "recipes"])
166
+
167
+ rel_score = compute_relevance_score(relevant, aiml_scoring_config)
168
+ irr_score = compute_relevance_score(irrelevant, aiml_scoring_config)
169
+ assert rel_score > irr_score
170
+
171
+
172
+# =============================================================================
173
+# 3. Rust Schema Validation
174
+# =============================================================================
175
+
176
+
177
+class TestRustSchemaValidation:
178
+ """Issue #71: Validate rust config passes schema validation."""
179
+
180
+ def test_rust_config_passes_validation(self, rust_config):
181
+ assert rust_config.topic.id == "rust"
182
+ assert rust_config.topic.name == "Rust Ecosystem"
183
+ assert len(rust_config.queries.primary) >= 2
184
+
185
+ def test_rust_scoring_section(self, rust_config):
186
+ assert rust_config.scoring.language_boost.get("Rust") == 1.3
187
+ assert rust_config.scoring.min_stars == 30
188
+ assert rust_config.scoring.max_age_days == 730
189
+
190
+ def test_rust_topic_relevance(self, rust_config):
191
+ assert "rust" in rust_config.scoring.topic_relevance
192
+ assert "cargo" in rust_config.scoring.topic_relevance
193
+ assert "wasm" in rust_config.scoring.topic_relevance
194
+
195
+ def test_rust_quality_section(self, rust_config):
196
+ assert rust_config.quality.min_repos_per_week == 5
197
+ assert rust_config.quality.max_repos_per_week == 25
198
+ assert rust_config.quality.min_quality_score == 55
199
+
200
+
201
+# =============================================================================
202
+# 4. Rust Scoring Pipeline
203
+# =============================================================================
204
+
205
+
206
+class TestRustScoringPipeline:
207
+ """Issue #71: Rust repos score reasonably with language_boost."""
208
+
209
+ def test_typical_rust_repos_score_above_threshold(self, rust_scoring_config, rust_repos):
210
+ scored = score_repos(rust_repos, rust_scoring_config)
211
+ # Most Rust repos should pass the lower threshold (35)
212
+ assert len(scored) >= 3
213
+
214
+ def test_rust_language_boost_applies(self, rust_scoring_config):
215
+ rust_repo = _make_repo("cli-tool", "Rust", 300, 60, ["rust", "cli"])
216
+ go_repo = _make_repo("cli-tool-go", "Go", 300, 60, ["go", "cli"])
217
+
218
+ rust_score = compute_relevance_score(rust_repo, rust_scoring_config)
219
+ go_score = compute_relevance_score(go_repo, rust_scoring_config)
220
+ assert rust_score > go_score
221
+
222
+ def test_narrower_topic_still_produces_results(self, rust_scoring_config, rust_repos):
223
+ """Rust is narrower than AI/ML but should still yield min_repos_per_week."""
224
+ min_required = 5 # from config quality.min_repos_per_week
225
+ scored = score_repos(rust_repos, rust_scoring_config)
226
+ assert len(scored) >= min_required
227
+
228
+ def test_high_quality_rust_repo_scores_above_40(self, rust_scoring_config):
229
+ repo = _make_repo("blazing-fast", "Rust", 1000, 150, ["rust", "async-rust", "cargo"])
230
+ score = compute_relevance_score(repo, rust_scoring_config)
231
+ assert score >= 40
232
+
233
+
234
+# =============================================================================
235
+# 5. Cross-Topic Isolation
236
+# =============================================================================
237
+
238
+
239
+class TestCrossTopicIsolation:
240
+ """Verify configs don't score repos from other domains highly."""
241
+
242
+ def test_aiml_config_does_not_score_rust_repos_highly(self, aiml_scoring_config, rust_repos):
243
+ """Rust repos should score lower under ai-ml config due to no topic overlap."""
244
+ scored = score_repos(rust_repos, aiml_scoring_config)
245
+ # Rust repos lack AI/ML topics, so fewer should pass threshold
246
+ for repo in scored:
247
+ # No Rust repo should score as high as a good AI/ML repo would
248
+ assert repo["relevance_score"] < 70
249
+
250
+ def test_rust_config_does_not_score_python_ml_repos_highly(self, rust_scoring_config, aiml_repos):
251
+ """Python ML repos should score lower under rust config due to topic/lang mismatch."""
252
+ scored = score_repos(aiml_repos, rust_scoring_config)
253
+ # Python repos don't get Rust language boost and lack rust topics
254
+ for repo in scored:
255
+ assert repo["relevance_score"] < 75
256
+
257
+ def test_aiml_repos_score_higher_with_own_config(self, aiml_scoring_config, rust_scoring_config, aiml_repos):
258
+ """AI/ML repos should score higher with ai-ml config than rust config."""
259
+ aiml_scored = score_repos(aiml_repos, aiml_scoring_config)
260
+ rust_scored = score_repos(aiml_repos, rust_scoring_config)
261
+
262
+ avg_aiml = sum(r["relevance_score"] for r in aiml_scored) / max(len(aiml_scored), 1)
263
+ avg_rust = sum(r["relevance_score"] for r in rust_scored) / max(len(rust_scored), 1) if rust_scored else 0
264
+ assert avg_aiml > avg_rust
265
+
266
+ def test_rust_repos_score_higher_with_own_config(self, aiml_scoring_config, rust_scoring_config, rust_repos):
267
+ """Rust repos should score higher with rust config than ai-ml config."""
268
+ rust_scored = score_repos(rust_repos, rust_scoring_config)
269
+ aiml_scored = score_repos(rust_repos, aiml_scoring_config)
270
+
271
+ avg_rust = sum(r["relevance_score"] for r in rust_scored) / max(len(rust_scored), 1)
272
+ avg_aiml = sum(r["relevance_score"] for r in aiml_scored) / max(len(aiml_scored), 1) if aiml_scored else 0
273
+ assert avg_rust > avg_aiml