| 1 | #!/usr/bin/env python3 |
| 2 | """End-to-end integration test for the SquadScope topic pipeline with ai-ml config. |
| 3 | |
| 4 | Exercises the full pipeline: validate → crawl (mocked) → score → quality gate → |
| 5 | prediction ledger → content verification, using realistic AI/ML mock data. |
| 6 | """ |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import json |
| 11 | import os |
| 12 | import shutil |
| 13 | import tempfile |
| 14 | import unittest |
| 15 | from pathlib import Path |
| 16 | |
| 17 | REPO_ROOT = Path(__file__).resolve().parent.parent |
| 18 | |
| 19 | MOCK_REPOS = [ |
| 20 | { |
| 21 | "full_name": "org/pytorch-trainer", |
| 22 | "language": "Python", |
| 23 | "topics": ["machine-learning", "pytorch", "deep-learning"], |
| 24 | "stars": 500, |
| 25 | "stars_gained": 50, |
| 26 | "forks": 80, |
| 27 | "created_at": "2024-06-01T00:00:00Z", |
| 28 | "description": "High-performance PyTorch training framework", |
| 29 | }, |
| 30 | { |
| 31 | "full_name": "org/llm-finetune", |
| 32 | "language": "Python", |
| 33 | "topics": ["llm", "transformers", "machine-learning"], |
| 34 | "stars": 1200, |
| 35 | "stars_gained": 200, |
| 36 | "forks": 150, |
| 37 | "created_at": "2024-03-15T00:00:00Z", |
| 38 | "description": "Fine-tuning toolkit for large language models", |
| 39 | }, |
| 40 | { |
| 41 | "full_name": "org/neural-notebook", |
| 42 | "language": "Jupyter Notebook", |
| 43 | "topics": ["deep-learning", "neural-network"], |
| 44 | "stars": 80, |
| 45 | "stars_gained": 30, |
| 46 | "forks": 12, |
| 47 | "created_at": "2025-01-10T00:00:00Z", |
| 48 | "description": "Interactive notebooks for neural network experiments", |
| 49 | }, |
| 50 | { |
| 51 | "full_name": "org/ai-image-gen", |
| 52 | "language": "Python", |
| 53 | "topics": ["artificial-intelligence", "deep-learning", "transformers"], |
| 54 | "stars": 3000, |
| 55 | "stars_gained": 400, |
| 56 | "forks": 500, |
| 57 | "created_at": "2023-11-20T00:00:00Z", |
| 58 | "description": "State-of-the-art image generation with diffusion models", |
| 59 | }, |
| 60 | { |
| 61 | "full_name": "org/ml-pipeline", |
| 62 | "language": "Python", |
| 63 | "topics": ["machine-learning", "mlops"], |
| 64 | "stars": 250, |
| 65 | "stars_gained": 35, |
| 66 | "forks": 40, |
| 67 | "created_at": "2024-09-01T00:00:00Z", |
| 68 | "description": "End-to-end ML pipeline orchestration", |
| 69 | }, |
| 70 | { |
| 71 | "full_name": "org/transformer-serving", |
| 72 | "language": "Python", |
| 73 | "topics": ["transformers", "llm", "inference"], |
| 74 | "stars": 900, |
| 75 | "stars_gained": 120, |
| 76 | "forks": 95, |
| 77 | "created_at": "2024-05-01T00:00:00Z", |
| 78 | "description": "Scalable transformer model serving", |
| 79 | }, |
| 80 | { |
| 81 | "full_name": "org/rust-cli-tool", |
| 82 | "language": "Rust", |
| 83 | "topics": ["cli-tool", "developer-tools"], |
| 84 | "stars": 300, |
| 85 | "stars_gained": 15, |
| 86 | "forks": 20, |
| 87 | "created_at": "2024-07-01T00:00:00Z", |
| 88 | "description": "Fast CLI utility written in Rust", |
| 89 | }, |
| 90 | { |
| 91 | "full_name": "org/web-dashboard", |
| 92 | "language": "JavaScript", |
| 93 | "topics": ["web", "dashboard", "react"], |
| 94 | "stars": 400, |
| 95 | "stars_gained": 25, |
| 96 | "forks": 50, |
| 97 | "created_at": "2024-04-01T00:00:00Z", |
| 98 | "description": "Modern web dashboard framework", |
| 99 | }, |
| 100 | { |
| 101 | "full_name": "org/tiny-nn", |
| 102 | "language": "Python", |
| 103 | "topics": ["neural-network", "deep-learning"], |
| 104 | "stars": 150, |
| 105 | "stars_gained": 45, |
| 106 | "forks": 25, |
| 107 | "created_at": "2025-02-01T00:00:00Z", |
| 108 | "description": "Minimal neural network library for education", |
| 109 | }, |
| 110 | { |
| 111 | "full_name": "org/data-viz", |
| 112 | "language": "JavaScript", |
| 113 | "topics": ["visualization", "charts"], |
| 114 | "stars": 600, |
| 115 | "stars_gained": 20, |
| 116 | "forks": 70, |
| 117 | "created_at": "2023-06-01T00:00:00Z", |
| 118 | "description": "Data visualization library", |
| 119 | }, |
| 120 | ] |
| 121 | |
| 122 | # Raw data format expected by prediction_ledger |
| 123 | MOCK_RAW_DATA = { |
| 124 | "week": "2025-W25", |
| 125 | "new_repos": MOCK_REPOS[:5], |
| 126 | "trending_repos": MOCK_REPOS[5:], |
| 127 | } |
| 128 | |
| 129 | MOCK_SUMMARY_CONTENT = """--- |
| 130 | topic: ai-ml |
| 131 | week: 2025-W25 |
| 132 | repos_scored: 8 |
| 133 | --- |
| 134 | |
| 135 | # AI & ML Weekly Digest — 2025-W25 |
| 136 | |
| 137 | ## Highlights |
| 138 | |
| 139 | - [org/pytorch-trainer](https://github.com/org/pytorch-trainer) — High-performance training |
| 140 | - [org/llm-finetune](https://github.com/org/llm-finetune) — LLM fine-tuning toolkit |
| 141 | - [org/ai-image-gen](https://github.com/org/ai-image-gen) — Diffusion image generation |
| 142 | - [org/transformer-serving](https://github.com/org/transformer-serving) — Model serving at scale |
| 143 | """ |
| 144 | |
| 145 | |
| 146 | class TestEndToEndTopicPipeline(unittest.TestCase): |
| 147 | """Full pipeline integration test with ai-ml topic config.""" |
| 148 | |
| 149 | def setUp(self): |
| 150 | """Create temp directory structure with ai-ml config and mock data.""" |
| 151 | self.temp_dir = tempfile.TemporaryDirectory() |
| 152 | self.work_dir = Path(self.temp_dir.name) |
| 153 | |
| 154 | # Copy the real ai-ml config |
| 155 | src_config = REPO_ROOT / "squadscope.topic.yml" |
| 156 | self.config_path = self.work_dir / "squadscope.topic.yml" |
| 157 | shutil.copy(src_config, self.config_path) |
| 158 | |
| 159 | # Create data directories |
| 160 | self.raw_path = self.work_dir / "data" / "raw" / "ai-ml" |
| 161 | self.analyzed_path = self.work_dir / "data" / "analyzed" / "ai-ml" |
| 162 | self.metrics_path = self.work_dir / "data" / "metrics" / "ai-ml" |
| 163 | for d in (self.raw_path, self.analyzed_path, self.metrics_path): |
| 164 | d.mkdir(parents=True, exist_ok=True) |
| 165 | |
| 166 | # Write mock raw crawl data (list format for score_repos) |
| 167 | self.raw_json_path = self.raw_path / "2025-W25.json" |
| 168 | self.raw_json_path.write_text(json.dumps(MOCK_REPOS, indent=2), encoding="utf-8") |
| 169 | |
| 170 | # Write raw data in dict format for prediction_ledger |
| 171 | self.raw_dict_path = self.raw_path / "2025-W25-full.json" |
| 172 | self.raw_dict_path.write_text(json.dumps(MOCK_RAW_DATA, indent=2), encoding="utf-8") |
| 173 | |
| 174 | # Write mock summary for prediction ledger |
| 175 | self.summary_path = self.analyzed_path / "2025-W25-summary.md" |
| 176 | self.summary_path.write_text(MOCK_SUMMARY_CONTENT, encoding="utf-8") |
| 177 | |
| 178 | # Change to work dir so relative paths in scripts work |
| 179 | self._orig_cwd = os.getcwd() |
| 180 | os.chdir(self.work_dir) |
| 181 | |
| 182 | def tearDown(self): |
| 183 | os.chdir(self._orig_cwd) |
| 184 | self.temp_dir.cleanup() |
| 185 | |
| 186 | def test_full_pipeline_ai_ml(self): |
| 187 | """Run the full pipeline: validate → score → quality gate → predictions.""" |
| 188 | import sys |
| 189 | |
| 190 | sys.path.insert(0, str(REPO_ROOT)) |
| 191 | |
| 192 | from scripts.prediction_ledger import append_predictions, generate_predictions |
| 193 | from scripts.quality_gate import check_quality, get_quality_config, write_metric |
| 194 | from scripts.score_repos import get_scoring_config, load_config, score_repos |
| 195 | from scripts.validate_topic_config import validate_file |
| 196 | |
| 197 | # --- Stage 1: Validate config --- |
| 198 | config_model = validate_file(str(self.config_path)) |
| 199 | self.assertEqual(config_model.topic.id, "ai-ml") |
| 200 | self.assertEqual(config_model.topic.name, "AI & Machine Learning") |
| 201 | self.assertGreater(len(config_model.queries.primary), 0) |
| 202 | self.assertIn("Python", config_model.scoring.language_boost) |
| 203 | |
| 204 | # --- Stage 2: Mock crawl data already created in setUp --- |
| 205 | raw_repos = json.loads(self.raw_json_path.read_text(encoding="utf-8")) |
| 206 | self.assertEqual(len(raw_repos), 10) |
| 207 | |
| 208 | # --- Stage 3: Score repos --- |
| 209 | config = load_config(self.config_path) |
| 210 | scoring_config = get_scoring_config(config) |
| 211 | scored = score_repos(raw_repos, scoring_config) |
| 212 | |
| 213 | # Verify scoring filters and ranks correctly |
| 214 | self.assertGreater(len(scored), 0) |
| 215 | self.assertLessEqual(len(scored), len(raw_repos)) |
| 216 | |
| 217 | # All scored repos should meet minimum relevance threshold |
| 218 | min_score = scoring_config["min_relevance_score"] |
| 219 | for repo in scored: |
| 220 | self.assertGreaterEqual(repo["relevance_score"], min_score) |
| 221 | self.assertIn("relevance_score", repo) |
| 222 | |
| 223 | # Verify sorted descending |
| 224 | scores = [r["relevance_score"] for r in scored] |
| 225 | self.assertEqual(scores, sorted(scores, reverse=True)) |
| 226 | |
| 227 | # AI/ML repos should score higher than non-AI repos |
| 228 | ai_repos = [ |
| 229 | r |
| 230 | for r in scored |
| 231 | if r["full_name"] in ("org/pytorch-trainer", "org/llm-finetune", "org/ai-image-gen") |
| 232 | ] |
| 233 | non_ai_repos = [ |
| 234 | r |
| 235 | for r in scored |
| 236 | if r["full_name"] in ("org/rust-cli-tool", "org/web-dashboard", "org/data-viz") |
| 237 | ] |
| 238 | if ai_repos and non_ai_repos: |
| 239 | max_non_ai = max(r["relevance_score"] for r in non_ai_repos) |
| 240 | min_ai = min(r["relevance_score"] for r in ai_repos) |
| 241 | self.assertGreater(min_ai, max_non_ai, "AI/ML repos should outscore non-AI repos") |
| 242 | |
| 243 | # Write scored output for quality gate |
| 244 | scored_path = self.work_dir / "scored.json" |
| 245 | scored_path.write_text(json.dumps(scored, indent=2), encoding="utf-8") |
| 246 | |
| 247 | # --- Stage 4: Quality gate --- |
| 248 | quality_config = get_quality_config(config) |
| 249 | scoring_cfg = config.get("scoring", {}) |
| 250 | metric = check_quality(scored, quality_config, scoring_cfg) |
| 251 | |
| 252 | self.assertIn("repos_scored", metric) |
| 253 | self.assertIn("repos_passing", metric) |
| 254 | self.assertIn("status", metric) |
| 255 | self.assertIn("warnings", metric) |
| 256 | self.assertGreater(metric["repos_scored"], 0) |
| 257 | |
| 258 | # Write metric file |
| 259 | week = "2025-W25" |
| 260 | metric_path = write_metric("ai-ml", metric, week) |
| 261 | self.assertTrue(metric_path.exists()) |
| 262 | metric_data = json.loads(metric_path.read_text(encoding="utf-8")) |
| 263 | self.assertEqual(metric_data["topic"], "ai-ml") |
| 264 | self.assertEqual(metric_data["week"], week) |
| 265 | |
| 266 | # --- Stage 5: Prediction ledger --- |
| 267 | summary_content = self.summary_path.read_text(encoding="utf-8") |
| 268 | raw_data = json.loads(self.raw_dict_path.read_text(encoding="utf-8")) |
| 269 | predictions = generate_predictions(summary_content, raw_data, week) |
| 270 | |
| 271 | self.assertGreaterEqual(len(predictions), 1) |
| 272 | self.assertLessEqual(len(predictions), 5) |
| 273 | |
| 274 | for pred in predictions: |
| 275 | self.assertIn("week", pred) |
| 276 | self.assertIn("repo", pred) |
| 277 | self.assertIn("prediction", pred) |
| 278 | self.assertIn("confidence", pred) |
| 279 | self.assertIn("reason", pred) |
| 280 | self.assertEqual(pred["week"], week) |
| 281 | self.assertGreater(pred["confidence"], 0) |
| 282 | self.assertIn( |
| 283 | pred["prediction"], |
| 284 | [ |
| 285 | "rising_star", |
| 286 | "emerging_topic", |
| 287 | "momentum_shift", |
| 288 | "breakout_candidate", |
| 289 | "declining_signal", |
| 290 | ], |
| 291 | ) |
| 292 | |
| 293 | # Write predictions to ledger |
| 294 | predictions_path = self.metrics_path / "predictions.jsonl" |
| 295 | append_predictions(predictions, predictions_path) |
| 296 | self.assertTrue(predictions_path.exists()) |
| 297 | |
| 298 | # Verify JSONL format |
| 299 | lines = predictions_path.read_text(encoding="utf-8").strip().split("\n") |
| 300 | self.assertEqual(len(lines), len(predictions)) |
| 301 | for line in lines: |
| 302 | entry = json.loads(line) |
| 303 | self.assertIn("repo", entry) |
| 304 | self.assertIn("prediction", entry) |
| 305 | |
| 306 | # --- Stage 6: Content generation verification --- |
| 307 | # Verify scored data is suitable for content generation |
| 308 | top_repos = scored[:5] |
| 309 | for repo in top_repos: |
| 310 | self.assertIn("full_name", repo) |
| 311 | self.assertIn("description", repo) |
| 312 | self.assertIn("relevance_score", repo) |
| 313 | # Verify topic-specific content (not generic) |
| 314 | topics = repo.get("topics", []) |
| 315 | ai_topics = { |
| 316 | "machine-learning", |
| 317 | "deep-learning", |
| 318 | "artificial-intelligence", |
| 319 | "neural-network", |
| 320 | "llm", |
| 321 | "transformers", |
| 322 | } |
| 323 | has_ai_topic = bool(set(t.lower() for t in topics) & ai_topics) |
| 324 | self.assertTrue(has_ai_topic, f"Top repo {repo['full_name']} should have AI/ML topics") |
| 325 | |
| 326 | def test_config_validation_rejects_invalid(self): |
| 327 | """Ensure validate rejects malformed configs.""" |
| 328 | import sys |
| 329 | |
| 330 | sys.path.insert(0, str(REPO_ROOT)) |
| 331 | from scripts.validate_topic_config import validate_file |
| 332 | |
| 333 | # Write an invalid config (missing required fields) |
| 334 | bad_config = self.work_dir / "bad.yml" |
| 335 | bad_config.write_text("topic:\n id: INVALID ID WITH SPACES\n", encoding="utf-8") |
| 336 | |
| 337 | with self.assertRaises(Exception): |
| 338 | validate_file(str(bad_config)) |
| 339 | |
| 340 | def test_scoring_respects_language_boost(self): |
| 341 | """Verify Python repos get a language boost over unlisted languages.""" |
| 342 | import sys |
| 343 | |
| 344 | sys.path.insert(0, str(REPO_ROOT)) |
| 345 | from scripts.score_repos import compute_relevance_score, get_scoring_config, load_config |
| 346 | |
| 347 | config = load_config(self.config_path) |
| 348 | scoring_config = get_scoring_config(config) |
| 349 | |
| 350 | base_repo = { |
| 351 | "full_name": "test/repo", |
| 352 | "stars": 500, |
| 353 | "stars_gained": 50, |
| 354 | "topics": ["machine-learning"], |
| 355 | "created_at": "2024-06-01T00:00:00Z", |
| 356 | } |
| 357 | |
| 358 | python_repo = {**base_repo, "language": "Python"} |
| 359 | rust_repo = {**base_repo, "language": "Rust"} |
| 360 | |
| 361 | python_score = compute_relevance_score(python_repo, scoring_config) |
| 362 | rust_score = compute_relevance_score(rust_repo, scoring_config) |
| 363 | |
| 364 | self.assertGreater( |
| 365 | python_score, rust_score, "Python should score higher with language_boost configured" |
| 366 | ) |
| 367 | |
| 368 | |
| 369 | if __name__ == "__main__": |
| 370 | unittest.main() |