@cryptotaxi247 / SquadScope-1 / commits / 4436ebf

feat: implement repo scoring pipeline (#61) (#95)

* feat: add prediction ledger output (#64) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * feat: implement repo scoring pipeline (#61) Add scripts/score_repos.py that scores crawled repos 0-100 on topic relevance using star count, velocity, language match, topic overlap, and age signals. Reads config from squadscope.topic.yml scoring section and filters repos below min_relevance_score threshold. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Juan Manuel Servera committed May 19, 2026 at 15:45 UTC 4436ebf24b8dab34377b17e659eed1b4ee5ac20b
4 files changed +1206
scripts/prediction_ledger.py new
+305
@@ -0,0 +1,305 @@
1 +#!/usr/bin/env python3
2 +"""Generate prediction ledger entries from analyzed summaries and raw data.
3 +
4 +Reads an analyzed summary markdown and its corresponding raw JSON to produce
5 +3-5 heuristic predictions about repos likely to gain momentum. Predictions
6 +are appended to a per-topic JSONL file for later validation.
7 +
8 +Usage:
9 + python scripts/prediction_ledger.py [--input FILE] [--topic TOPIC] [--raw FILE]
10 +"""
11 +
12 +from __future__ import annotations
13 +
14 +import argparse
15 +import json
16 +import re
17 +from pathlib import Path
18 +from typing import Any
19 +
20 +from scripts.topic_paths import analyzed_dir, metrics_dir, raw_dir
21 +
22 +FRONTMATTER_PATTERN = re.compile(r"^---\n(.*?)\n---\n(.*)\Z", re.DOTALL)
23 +WEEK_PATTERN = re.compile(r"\d{4}-W\d{2}")
24 +REPO_LINK_PATTERN = re.compile(
25 + r"\[(?P<full_name>[^\]]+/[^\]]+)\]\(https://github\.com/[^\)]+\)"
26 +)
27 +
28 +PREDICTION_TYPES = [
29 + "rising_star",
30 + "emerging_topic",
31 + "momentum_shift",
32 + "breakout_candidate",
33 + "declining_signal",
34 +]
35 +
36 +MAX_PREDICTIONS = 5
37 +MIN_PREDICTIONS = 3
38 +
39 +
40 +def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
41 + parser = argparse.ArgumentParser(
42 + description="Generate prediction ledger entries from analyzed summaries."
43 + )
44 + parser.add_argument(
45 + "--input",
46 + default=None,
47 + help="Path to analyzed summary markdown (data/analyzed/{topic}/YYYY-WNN-summary.md).",
48 + )
49 + parser.add_argument(
50 + "--topic",
51 + default=None,
52 + help="Topic ID for path resolution. Defaults to general.",
53 + )
54 + parser.add_argument(
55 + "--raw",
56 + default=None,
57 + help="Path to raw JSON data. Inferred from summary week if not given.",
58 + )
59 + return parser.parse_args(argv)
60 +
61 +
62 +def find_latest_summary(topic_id: str | None) -> Path:
63 + """Find the most recent analyzed summary for a topic."""
64 + search = analyzed_dir(topic_id)
65 + candidates = sorted(search.glob("*-summary.md"))
66 + if not candidates:
67 + raise FileNotFoundError(f"No summaries found in {search}")
68 + return candidates[-1]
69 +
70 +
71 +def extract_week(text: str) -> str | None:
72 + """Extract YYYY-WNN week identifier from text."""
73 + match = WEEK_PATTERN.search(text)
74 + return match.group(0) if match else None
75 +
76 +
77 +def infer_raw_path(summary_path: Path, topic_id: str | None) -> Path:
78 + """Infer the raw JSON path from the summary filename."""
79 + week = extract_week(summary_path.name)
80 + if not week:
81 + raise ValueError(f"Cannot infer week from {summary_path.name}")
82 + return raw_dir(topic_id) / f"{week}.json"
83 +
84 +
85 +def parse_summary(content: str) -> dict[str, Any]:
86 + """Parse an analyzed summary markdown into frontmatter and body."""
87 + match = FRONTMATTER_PATTERN.match(content)
88 + if not match:
89 + return {"frontmatter": {}, "body": content}
90 +
91 + fm_text, body = match.group(1), match.group(2)
92 + frontmatter: dict[str, Any] = {}
93 + for line in fm_text.splitlines():
94 + if ":" in line:
95 + key, _, value = line.partition(":")
96 + value = value.strip().strip('"').strip("'")
97 + frontmatter[key.strip()] = value
98 + return {"frontmatter": frontmatter, "body": body}
99 +
100 +
101 +def extract_repos_from_summary(body: str) -> list[str]:
102 + """Extract repo full_names mentioned in the summary body."""
103 + seen: set[str] = set()
104 + repos: list[str] = []
105 + for match in REPO_LINK_PATTERN.finditer(body):
106 + name = match.group("full_name")
107 + if name not in seen:
108 + seen.add(name)
109 + repos.append(name)
110 + return repos
111 +
112 +
113 +def load_raw_data(raw_path: Path) -> dict[str, Any]:
114 + """Load and return raw JSON data."""
115 + with open(raw_path, encoding="utf-8") as f:
116 + return json.load(f)
117 +
118 +
119 +def build_repo_index(raw_data: dict[str, Any]) -> dict[str, dict[str, Any]]:
120 + """Index repos from raw data by full_name for quick lookup."""
121 + index: dict[str, dict[str, Any]] = {}
122 + for section in ("new_repos", "trending_repos"):
123 + for repo in raw_data.get(section, []):
124 + full_name = repo.get("full_name", "")
125 + if full_name:
126 + entry = index.get(full_name, {})
127 + entry.update(repo)
128 + entry["_source"] = section
129 + index[full_name] = entry
130 + return index
131 +
132 +
133 +def score_rising_star(repo: dict[str, Any]) -> float:
134 + """Score a repo for rising_star potential."""
135 + stars = repo.get("stars", 0)
136 + is_new = repo.get("_source") == "new_repos"
137 + # High stars on a new repo is a strong signal
138 + if is_new and stars >= 1000:
139 + return min(0.9, 0.5 + (stars / 10000))
140 + if is_new and stars >= 100:
141 + return min(0.7, 0.3 + (stars / 5000))
142 + if stars >= 5000:
143 + return 0.4
144 + return 0.2
145 +
146 +
147 +def score_breakout_candidate(repo: dict[str, Any]) -> float:
148 + """Score a repo for breakout_candidate potential."""
149 + stars = repo.get("stars", 0)
150 + forks = repo.get("forks", 0)
151 + is_new = repo.get("_source") == "new_repos"
152 + fork_ratio = forks / max(stars, 1)
153 + if is_new and fork_ratio > 0.1 and stars >= 50:
154 + return min(0.8, 0.4 + fork_ratio)
155 + if stars >= 500 and fork_ratio > 0.15:
156 + return 0.6
157 + return 0.2
158 +
159 +
160 +def score_momentum_shift(repo: dict[str, Any]) -> float:
161 + """Score a repo for momentum_shift (trending but established)."""
162 + stars = repo.get("stars", 0)
163 + is_trending = repo.get("_source") == "trending_repos"
164 + if is_trending and stars >= 10000:
165 + return 0.6
166 + if is_trending and stars >= 1000:
167 + return 0.5
168 + return 0.2
169 +
170 +
171 +def classify_prediction(repo: dict[str, Any]) -> tuple[str, float, str]:
172 + """Classify a repo into a prediction type with confidence and reason."""
173 + scores = {
174 + "rising_star": score_rising_star(repo),
175 + "breakout_candidate": score_breakout_candidate(repo),
176 + "momentum_shift": score_momentum_shift(repo),
177 + }
178 +
179 + best_type = max(scores, key=scores.get) # type: ignore[arg-type]
180 + confidence = scores[best_type]
181 +
182 + reasons = {
183 + "rising_star": f"New repo with {repo.get('stars', 0)} stars and active development",
184 + "breakout_candidate": (
185 + f"High fork ratio ({repo.get('forks', 0)} forks / "
186 + f"{repo.get('stars', 0)} stars) suggests community adoption"
187 + ),
188 + "momentum_shift": (
189 + f"Established repo ({repo.get('stars', 0)} stars) trending this week"
190 + ),
191 + }
192 +
193 + return best_type, round(confidence, 2), reasons[best_type]
194 +
195 +
196 +def generate_predictions(
197 + summary_content: str,
198 + raw_data: dict[str, Any],
199 + week: str,
200 +) -> list[dict[str, Any]]:
201 + """Generate 3-5 predictions from analyzed summary and raw data."""
202 + parsed = parse_summary(summary_content)
203 + mentioned_repos = extract_repos_from_summary(parsed["body"])
204 + repo_index = build_repo_index(raw_data)
205 +
206 + predictions: list[dict[str, Any]] = []
207 +
208 + # Score mentioned repos that exist in raw data
209 + candidates: list[tuple[str, str, float, str]] = []
210 + for repo_name in mentioned_repos:
211 + if repo_name in repo_index:
212 + pred_type, confidence, reason = classify_prediction(repo_index[repo_name])
213 + candidates.append((repo_name, pred_type, confidence, reason))
214 +
215 + # Sort by confidence descending, take top entries
216 + candidates.sort(key=lambda x: x[2], reverse=True)
217 +
218 + for repo_name, pred_type, confidence, reason in candidates[:MAX_PREDICTIONS]:
219 + predictions.append(
220 + {
221 + "week": week,
222 + "repo": repo_name,
223 + "prediction": pred_type,
224 + "confidence": confidence,
225 + "reason": reason,
226 + "validated": None,
227 + }
228 + )
229 +
230 + # If we have fewer than MIN_PREDICTIONS from mentioned repos,
231 + # supplement from raw data's new_repos
232 + if len(predictions) < MIN_PREDICTIONS:
233 + existing = {p["repo"] for p in predictions}
234 + for repo in raw_data.get("new_repos", []):
235 + if len(predictions) >= MIN_PREDICTIONS:
236 + break
237 + full_name = repo.get("full_name", "")
238 + if full_name and full_name not in existing:
239 + pred_type, confidence, reason = classify_prediction(repo)
240 + if confidence >= 0.3:
241 + predictions.append(
242 + {
243 + "week": week,
244 + "repo": full_name,
245 + "prediction": pred_type,
246 + "confidence": confidence,
247 + "reason": reason,
248 + "validated": None,
249 + }
250 + )
251 + existing.add(full_name)
252 +
253 + return predictions[:MAX_PREDICTIONS]
254 +
255 +
256 +def append_predictions(predictions: list[dict[str, Any]], output_path: Path) -> None:
257 + """Append predictions to a JSONL file."""
258 + output_path.parent.mkdir(parents=True, exist_ok=True)
259 + with open(output_path, "a", encoding="utf-8") as f:
260 + for pred in predictions:
261 + f.write(json.dumps(pred, ensure_ascii=False) + "\n")
262 +
263 +
264 +def main(argv: list[str] | None = None) -> list[dict[str, Any]]:
265 + """Main entry point. Returns the generated predictions."""
266 + args = parse_args(argv)
267 + topic_id = args.topic
268 +
269 + # Resolve input summary
270 + if args.input:
271 + summary_path = Path(args.input)
272 + else:
273 + summary_path = find_latest_summary(topic_id)
274 +
275 + # Read summary
276 + summary_content = summary_path.read_text(encoding="utf-8")
277 +
278 + # Resolve raw data path
279 + if args.raw:
280 + raw_path = Path(args.raw)
281 + else:
282 + raw_path = infer_raw_path(summary_path, topic_id)
283 +
284 + raw_data = load_raw_data(raw_path)
285 +
286 + # Determine week
287 + week = extract_week(summary_path.name) or raw_data.get("week", "unknown")
288 +
289 + # Generate predictions
290 + predictions = generate_predictions(summary_content, raw_data, week)
291 +
292 + # Write output
293 + output_path = metrics_dir(topic_id) / "predictions.jsonl"
294 + append_predictions(predictions, output_path)
295 +
296 + # Print summary
297 + print(f"Generated {len(predictions)} predictions for {week}")
298 + for p in predictions:
299 + print(f" [{p['prediction']}] {p['repo']} (confidence: {p['confidence']})")
300 +
301 + return predictions
302 +
303 +
304 +if __name__ == "__main__":
305 + main()
scripts/score_repos.py new
+193
@@ -0,0 +1,193 @@
1 +#!/usr/bin/env python3
2 +"""Score crawled repositories on topic relevance for SquadScope.
3 +
4 +Reads raw crawl JSON, applies scoring criteria from squadscope.topic.yml,
5 +and outputs a ranked list of repos with relevance_score field.
6 +
7 +Usage:
8 + python scripts/score_repos.py [--config squadscope.topic.yml] \
9 + [--input data/raw/ai-ml/2026-W21.json] [--output scored.json] [--topic ai-ml]
10 +"""
11 +
12 +from __future__ import annotations
13 +
14 +import argparse
15 +import json
16 +import math
17 +import sys
18 +from datetime import UTC, datetime
19 +from pathlib import Path
20 +from typing import Any
21 +
22 +import yaml
23 +
24 +from scripts.topic_paths import load_topic_id, raw_dir
25 +
26 +
27 +def load_config(config_path: str | Path) -> dict[str, Any]:
28 + """Load and return the full config from a YAML file."""
29 + path = Path(config_path)
30 + if not path.exists():
31 + raise FileNotFoundError(f"Config file not found: {config_path}")
32 + with open(path, encoding="utf-8") as f:
33 + return yaml.safe_load(f) or {}
34 +
35 +
36 +def get_scoring_config(config: dict[str, Any]) -> dict[str, Any]:
37 + """Extract scoring section with defaults."""
38 + defaults = {
39 + "min_stars": 20,
40 + "min_stars_gained": 10,
41 + "max_age_days": 365,
42 + "min_relevance_score": 40,
43 + "language_boost": {},
44 + "topic_relevance": [],
45 + }
46 + scoring = config.get("scoring", {})
47 + return {**defaults, **scoring}
48 +
49 +
50 +def find_latest_raw_json(topic_id: str | None) -> Path | None:
51 + """Find the most recent raw JSON file for a topic."""
52 + directory = raw_dir(topic_id)
53 + if not directory.exists():
54 + return None
55 + json_files = sorted(directory.glob("*.json"), reverse=True)
56 + return json_files[0] if json_files else None
57 +
58 +
59 +def score_stars(stars: int) -> float:
60 + """Score based on star count with diminishing returns (0-25 points)."""
61 + if stars <= 0:
62 + return 0.0
63 + # Log scale: 100 stars ≈ 12.5, 1000 stars ≈ 18.7, 10000 stars ≈ 25
64 + return min(25.0, 25.0 * math.log10(stars) / math.log10(10000))
65 +
66 +
67 +def score_stars_gained(stars_gained: int) -> float:
68 + """Score based on stars gained velocity (0-25 points)."""
69 + if stars_gained <= 0:
70 + return 0.0
71 + # Log scale with faster saturation
72 + return min(25.0, 25.0 * math.log10(1 + stars_gained) / math.log10(1000))
73 +
74 +
75 +def score_language(language: str | None, language_boost: dict[str, float]) -> float:
76 + """Score based on language match (0-15 points)."""
77 + if not language or not language_boost:
78 + return 7.5 # neutral score when no language info or no config
79 + multiplier = language_boost.get(language, 1.0)
80 + return min(15.0, 7.5 * multiplier)
81 +
82 +
83 +def score_topics(repo_topics: list[str], topic_relevance: list[str]) -> float:
84 + """Score based on topic overlap (0-25 points)."""
85 + if not topic_relevance or not repo_topics:
86 + return 0.0
87 + repo_set = set(t.lower() for t in repo_topics)
88 + relevance_set = set(t.lower() for t in topic_relevance)
89 + matches = len(repo_set & relevance_set)
90 + max_possible = min(len(relevance_set), 3) # cap at 3 matches for full score
91 + if max_possible == 0:
92 + return 0.0
93 + return min(25.0, 25.0 * matches / max_possible)
94 +
95 +
96 +def score_age(created_at: str | None, max_age_days: int) -> float:
97 + """Score based on repo age (0-10 points). Newer repos get a boost."""
98 + if not created_at:
99 + return 5.0 # neutral when unknown
100 + try:
101 + created = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
102 + except (ValueError, TypeError):
103 + return 5.0
104 + now = datetime.now(UTC)
105 + age_days = (now - created).days
106 + if age_days < 0:
107 + age_days = 0
108 + if age_days > max_age_days:
109 + # Penalty for old repos: scale down from 5 to 0
110 + penalty_ratio = min((age_days - max_age_days) / max_age_days, 1.0)
111 + return max(0.0, 5.0 * (1.0 - penalty_ratio))
112 + # Boost for newer repos: 0 days = 10, max_age_days = 5
113 + freshness = 1.0 - (age_days / max_age_days)
114 + return 5.0 + 5.0 * freshness
115 +
116 +
117 +def compute_relevance_score(repo: dict[str, Any], scoring_config: dict[str, Any]) -> float:
118 + """Compute a 0-100 relevance score for a single repo."""
119 + stars = repo.get("stars", 0) or 0
120 + stars_gained = repo.get("stars_gained", 0) or 0
121 + language = repo.get("language")
122 + topics = repo.get("topics", []) or []
123 + created_at = repo.get("created_at")
124 +
125 + s_stars = score_stars(stars)
126 + s_gained = score_stars_gained(stars_gained)
127 + s_lang = score_language(language, scoring_config.get("language_boost", {}))
128 + s_topics = score_topics(topics, scoring_config.get("topic_relevance", []))
129 + s_age = score_age(created_at, scoring_config.get("max_age_days", 365))
130 +
131 + raw_score = s_stars + s_gained + s_lang + s_topics + s_age
132 + return round(min(100.0, max(0.0, raw_score)), 1)
133 +
134 +
135 +def score_repos(repos: list[dict[str, Any]], scoring_config: dict[str, Any]) -> list[dict[str, Any]]:
136 + """Score and filter a list of repos. Returns sorted list with relevance_score."""
137 + min_score = scoring_config.get("min_relevance_score", 40)
138 + scored = []
139 + for repo in repos:
140 + score = compute_relevance_score(repo, scoring_config)
141 + if score >= min_score:
142 + scored.append({**repo, "relevance_score": score})
143 + scored.sort(key=lambda r: r["relevance_score"], reverse=True)
144 + return scored
145 +
146 +
147 +def main(argv: list[str] | None = None) -> int:
148 + parser = argparse.ArgumentParser(description="Score repos on topic relevance")
149 + parser.add_argument("--config", default="squadscope.topic.yml", help="Path to topic config YAML")
150 + parser.add_argument("--input", default=None, help="Path to raw crawl JSON file")
151 + parser.add_argument("--output", default=None, help="Output file path (default: stdout)")
152 + parser.add_argument("--topic", default=None, help="Topic ID override")
153 + args = parser.parse_args(argv)
154 +
155 + config = load_config(args.config)
156 + scoring_config = get_scoring_config(config)
157 +
158 + topic_id = args.topic or load_topic_id(args.config)
159 +
160 + if args.input:
161 + input_path = Path(args.input)
162 + else:
163 + input_path = find_latest_raw_json(topic_id)
164 + if input_path is None:
165 + print(f"Error: No raw JSON found for topic '{topic_id}'", file=sys.stderr)
166 + return 1
167 +
168 + if not input_path.exists():
169 + print(f"Error: Input file not found: {input_path}", file=sys.stderr)
170 + return 1
171 +
172 + with open(input_path, encoding="utf-8") as f:
173 + repos = json.load(f)
174 +
175 + if not isinstance(repos, list):
176 + print("Error: Input JSON must be a list of repo objects", file=sys.stderr)
177 + return 1
178 +
179 + scored = score_repos(repos, scoring_config)
180 +
181 + output_json = json.dumps(scored, indent=2, ensure_ascii=False)
182 + if args.output:
183 + Path(args.output).parent.mkdir(parents=True, exist_ok=True)
184 + with open(args.output, "w", encoding="utf-8") as f:
185 + f.write(output_json + "\n")
186 + else:
187 + print(output_json)
188 +
189 + return 0
190 +
191 +
192 +if __name__ == "__main__":
193 + sys.exit(main())
tests/test_prediction_ledger.py new
+305
@@ -0,0 +1,305 @@
1 +"""Tests for the prediction ledger module."""
2 +
3 +from __future__ import annotations
4 +
5 +import json
6 +from pathlib import Path
7 +from unittest.mock import patch
8 +
9 +import pytest
10 +
11 +from scripts.prediction_ledger import (
12 + MAX_PREDICTIONS,
13 + MIN_PREDICTIONS,
14 + PREDICTION_TYPES,
15 + append_predictions,
16 + build_repo_index,
17 + classify_prediction,
18 + extract_repos_from_summary,
19 + extract_week,
20 + generate_predictions,
21 + infer_raw_path,
22 + main,
23 + parse_summary,
24 + score_breakout_candidate,
25 + score_momentum_shift,
26 + score_rising_star,
27 +)
28 +
29 +SAMPLE_SUMMARY = """\
30 +---
31 +title: "Week 21, 2026 Analysis"
32 +date: 2026-05-18T12:07:20.778+02:00
33 +week: "2026-W21"
34 +year: 2026
35 +tags: [ai, agents]
36 +top_repo: "vercel-labs/zero"
37 +quality_score: 76
38 +summary: "Strong week for agent tooling."
39 +---
40 +
41 +## Notable New Repositories
42 +
43 +[vercel-labs/zero](https://github.com/vercel-labs/zero) is the top new repo.
44 +Also notable: [org/rising-repo](https://github.com/org/rising-repo) and
45 +[bigcorp/established](https://github.com/bigcorp/established).
46 +"""
47 +
48 +SAMPLE_RAW = {
49 + "week": "2026-W21",
50 + "crawled_at": "2026-05-18T08:54:09Z",
51 + "new_repos": [
52 + {
53 + "name": "zero",
54 + "owner": "vercel-labs",
55 + "full_name": "vercel-labs/zero",
56 + "description": "Agent infra",
57 + "language": "TypeScript",
58 + "stars": 2500,
59 + "forks": 150,
60 + "created_at": "2026-05-10T00:00:00Z",
61 + "topics": ["ai", "agents"],
62 + "license": "MIT",
63 + "url": "https://github.com/vercel-labs/zero",
64 + },
65 + {
66 + "name": "rising-repo",
67 + "owner": "org",
68 + "full_name": "org/rising-repo",
69 + "description": "Hot new project",
70 + "language": "Python",
71 + "stars": 800,
72 + "forks": 120,
73 + "created_at": "2026-05-12T00:00:00Z",
74 + "topics": ["ml"],
75 + "license": "Apache-2.0",
76 + "url": "https://github.com/org/rising-repo",
77 + },
78 + ],
79 + "trending_repos": [
80 + {
81 + "name": "established",
82 + "owner": "bigcorp",
83 + "full_name": "bigcorp/established",
84 + "description": "Major framework",
85 + "language": "JavaScript",
86 + "stars": 50000,
87 + "forks": 8000,
88 + "created_at": "2020-01-01T00:00:00Z",
89 + "topics": ["framework"],
90 + "license": "MIT",
91 + "url": "https://github.com/bigcorp/established",
92 + },
93 + ],
94 + "signals": {"top_topics": ["ai", "agents"]},
95 + "metadata": {"api_calls_used": 10, "rate_limit_remaining": 4990},
96 +}
97 +
98 +
99 +class TestExtractWeek:
100 + def test_extracts_from_filename(self):
101 + assert extract_week("2026-W21-summary.md") == "2026-W21"
102 +
103 + def test_extracts_from_text(self):
104 + assert extract_week("week: 2025-W03") == "2025-W03"
105 +
106 + def test_returns_none_for_no_match(self):
107 + assert extract_week("no week here") is None
108 +
109 +
110 +class TestParseSummary:
111 + def test_parses_frontmatter(self):
112 + result = parse_summary(SAMPLE_SUMMARY)
113 + assert result["frontmatter"]["week"] == "2026-W21"
114 + assert result["frontmatter"]["top_repo"] == "vercel-labs/zero"
115 +
116 + def test_parses_body(self):
117 + result = parse_summary(SAMPLE_SUMMARY)
118 + assert "Notable New Repositories" in result["body"]
119 +
120 + def test_handles_no_frontmatter(self):
121 + result = parse_summary("# Just a header\nSome content.")
122 + assert result["frontmatter"] == {}
123 + assert "Just a header" in result["body"]
124 +
125 +
126 +class TestExtractRepos:
127 + def test_extracts_repo_links(self):
128 + body = parse_summary(SAMPLE_SUMMARY)["body"]
129 + repos = extract_repos_from_summary(body)
130 + assert "vercel-labs/zero" in repos
131 + assert "org/rising-repo" in repos
132 + assert "bigcorp/established" in repos
133 +
134 + def test_deduplicates(self):
135 + text = (
136 + "[a/b](https://github.com/a/b) and [a/b](https://github.com/a/b)"
137 + )
138 + repos = extract_repos_from_summary(text)
139 + assert repos == ["a/b"]
140 +
141 + def test_empty_for_no_links(self):
142 + assert extract_repos_from_summary("no repos here") == []
143 +
144 +
145 +class TestBuildRepoIndex:
146 + def test_indexes_by_full_name(self):
147 + index = build_repo_index(SAMPLE_RAW)
148 + assert "vercel-labs/zero" in index
149 + assert "bigcorp/established" in index
150 +
151 + def test_marks_source(self):
152 + index = build_repo_index(SAMPLE_RAW)
153 + assert index["vercel-labs/zero"]["_source"] == "new_repos"
154 + assert index["bigcorp/established"]["_source"] == "trending_repos"
155 +
156 + def test_empty_data(self):
157 + assert build_repo_index({}) == {}
158 +
159 +
160 +class TestScoring:
161 + def test_rising_star_high_stars_new(self):
162 + repo = {"stars": 3000, "_source": "new_repos"}
163 + score = score_rising_star(repo)
164 + assert 0.5 <= score <= 0.9
165 +
166 + def test_rising_star_low_stars(self):
167 + repo = {"stars": 10, "_source": "new_repos"}
168 + score = score_rising_star(repo)
169 + assert score < 0.4
170 +
171 + def test_breakout_high_fork_ratio(self):
172 + repo = {"stars": 500, "forks": 100, "_source": "new_repos"}
173 + score = score_breakout_candidate(repo)
174 + assert score >= 0.4
175 +
176 + def test_momentum_shift_trending_big(self):
177 + repo = {"stars": 50000, "_source": "trending_repos"}
178 + score = score_momentum_shift(repo)
179 + assert score >= 0.5
180 +
181 + def test_momentum_shift_not_trending(self):
182 + repo = {"stars": 50000, "_source": "new_repos"}
183 + score = score_momentum_shift(repo)
184 + assert score < 0.3
185 +
186 +
187 +class TestClassifyPrediction:
188 + def test_returns_valid_type(self):
189 + repo = {"stars": 3000, "forks": 100, "_source": "new_repos"}
190 + pred_type, confidence, reason = classify_prediction(repo)
191 + assert pred_type in PREDICTION_TYPES
192 + assert 0.0 <= confidence <= 1.0
193 + assert len(reason) > 0
194 +
195 + def test_new_high_star_is_rising_star(self):
196 + repo = {"stars": 5000, "forks": 50, "_source": "new_repos"}
197 + pred_type, _, _ = classify_prediction(repo)
198 + assert pred_type == "rising_star"
199 +
200 + def test_trending_established_is_momentum(self):
201 + repo = {"stars": 50000, "forks": 1000, "_source": "trending_repos"}
202 + pred_type, _, _ = classify_prediction(repo)
203 + assert pred_type == "momentum_shift"
204 +
205 +
206 +class TestGeneratePredictions:
207 + def test_generates_predictions(self):
208 + preds = generate_predictions(SAMPLE_SUMMARY, SAMPLE_RAW, "2026-W21")
209 + assert MIN_PREDICTIONS <= len(preds) <= MAX_PREDICTIONS
210 +
211 + def test_prediction_structure(self):
212 + preds = generate_predictions(SAMPLE_SUMMARY, SAMPLE_RAW, "2026-W21")
213 + for p in preds:
214 + assert p["week"] == "2026-W21"
215 + assert p["prediction"] in PREDICTION_TYPES
216 + assert 0.0 <= p["confidence"] <= 1.0
217 + assert p["validated"] is None
218 + assert "repo" in p
219 + assert "reason" in p
220 +
221 + def test_sorted_by_confidence(self):
222 + preds = generate_predictions(SAMPLE_SUMMARY, SAMPLE_RAW, "2026-W21")
223 + confidences = [p["confidence"] for p in preds]
224 + assert confidences == sorted(confidences, reverse=True)
225 +
226 +
227 +class TestAppendPredictions:
228 + def test_appends_to_file(self, tmp_path):
229 + output = tmp_path / "predictions.jsonl"
230 + preds = [
231 + {
232 + "week": "2026-W21",
233 + "repo": "a/b",
234 + "prediction": "rising_star",
235 + "confidence": 0.7,
236 + "reason": "test",
237 + "validated": None,
238 + }
239 + ]
240 + append_predictions(preds, output)
241 + lines = output.read_text().strip().splitlines()
242 + assert len(lines) == 1
243 + assert json.loads(lines[0])["repo"] == "a/b"
244 +
245 + def test_appends_multiple_calls(self, tmp_path):
246 + output = tmp_path / "predictions.jsonl"
247 + pred = {
248 + "week": "2026-W21",
249 + "repo": "x/y",
250 + "prediction": "rising_star",
251 + "confidence": 0.5,
252 + "reason": "r",
253 + "validated": None,
254 + }
255 + append_predictions([pred], output)
256 + append_predictions([pred], output)
257 + lines = output.read_text().strip().splitlines()
258 + assert len(lines) == 2
259 +
260 + def test_creates_parent_dirs(self, tmp_path):
261 + output = tmp_path / "nested" / "dir" / "predictions.jsonl"
262 + append_predictions([], output)
263 + assert output.parent.exists()
264 +
265 +
266 +class TestInferRawPath:
267 + def test_infers_from_summary(self):
268 + summary = Path("data/analyzed/2026-W21-summary.md")
269 + raw = infer_raw_path(summary, None)
270 + assert raw == Path("data/raw/2026-W21.json")
271 +
272 + def test_infers_with_topic(self):
273 + summary = Path("data/analyzed/ai-ml/2026-W21-summary.md")
274 + raw = infer_raw_path(summary, "ai-ml")
275 + assert raw == Path("data/raw/ai-ml/2026-W21.json")
276 +
277 + def test_raises_on_no_week(self):
278 + with pytest.raises(ValueError):
279 + infer_raw_path(Path("bad-name.md"), None)
280 +
281 +
282 +class TestMain:
283 + def test_end_to_end(self, tmp_path):
284 + # Set up files
285 + summary_path = tmp_path / "analyzed" / "2026-W21-summary.md"
286 + summary_path.parent.mkdir(parents=True)
287 + summary_path.write_text(SAMPLE_SUMMARY)
288 +
289 + raw_path = tmp_path / "raw" / "2026-W21.json"
290 + raw_path.parent.mkdir(parents=True)
291 + raw_path.write_text(json.dumps(SAMPLE_RAW))
292 +
293 + metrics_path = tmp_path / "metrics"
294 +
295 + with patch("scripts.prediction_ledger.metrics_dir", return_value=metrics_path):
296 + preds = main([
297 + "--input", str(summary_path),
298 + "--raw", str(raw_path),
299 + ])
300 +
301 + assert len(preds) >= MIN_PREDICTIONS
302 + output_file = metrics_path / "predictions.jsonl"
303 + assert output_file.exists()
304 + lines = output_file.read_text().strip().splitlines()
305 + assert len(lines) == len(preds)
tests/test_score_repos.py new
+403
@@ -0,0 +1,403 @@
1 +"""Tests for scripts/score_repos.py scoring pipeline."""
2 +
3 +from __future__ import annotations
4 +
5 +import json
6 +import math
7 +from datetime import UTC, datetime, timedelta
8 +from pathlib import Path
9 +from unittest.mock import patch
10 +
11 +import pytest
12 +
13 +from scripts.score_repos import (
14 + compute_relevance_score,
15 + find_latest_raw_json,
16 + get_scoring_config,
17 + load_config,
18 + main,
19 + score_age,
20 + score_language,
21 + score_repos,
22 + score_stars,
23 + score_stars_gained,
24 + score_topics,
25 +)
26 +
27 +
28 +# --- Fixtures ---
29 +
30 +
31 +@pytest.fixture
32 +def scoring_config():
33 + """Default scoring config matching squadscope.topic.yml."""
34 + return {
35 + "min_stars": 20,
36 + "min_stars_gained": 10,
37 + "max_age_days": 365,
38 + "min_relevance_score": 40,
39 + "language_boost": {"Python": 1.2, "Jupyter Notebook": 1.1},
40 + "topic_relevance": [
41 + "machine-learning",
42 + "deep-learning",
43 + "artificial-intelligence",
44 + "neural-network",
45 + "llm",
46 + "transformers",
47 + ],
48 + }
49 +
50 +
51 +@pytest.fixture
52 +def sample_repo():
53 + """A typical high-quality AI/ML repo record."""
54 + return {
55 + "name": "awesome-ml",
56 + "owner": "researcher",
57 + "full_name": "researcher/awesome-ml",
58 + "description": "A machine learning framework",
59 + "language": "Python",
60 + "stars": 500,
61 + "forks": 50,
62 + "created_at": (datetime.now(UTC) - timedelta(days=30)).isoformat(),
63 + "topics": ["machine-learning", "deep-learning", "python"],
64 + "license": "MIT",
65 + "url": "https://github.com/researcher/awesome-ml",
66 + "stars_gained": 100,
67 + }
68 +
69 +
70 +@pytest.fixture
71 +def config_file(tmp_path):
72 + """Create a temporary config YAML file."""
73 + config = {
74 + "topic": {
75 + "id": "ai-ml",
76 + "name": "AI & ML",
77 + "description": "Test topic",
78 + },
79 + "queries": {"primary": ["topic:machine-learning"]},
80 + "scoring": {
81 + "min_stars": 20,
82 + "min_stars_gained": 10,
83 + "max_age_days": 365,
84 + "min_relevance_score": 40,
85 + "language_boost": {"Python": 1.2},
86 + "topic_relevance": ["machine-learning", "deep-learning"],
87 + },
88 + }
89 + path = tmp_path / "test_config.yml"
90 + import yaml
91 +
92 + path.write_text(yaml.dump(config), encoding="utf-8")
93 + return path
94 +
95 +
96 +# --- Test score_stars ---
97 +
98 +
99 +class TestScoreStars:
100 + def test_zero_stars(self):
101 + assert score_stars(0) == 0.0
102 +
103 + def test_negative_stars(self):
104 + assert score_stars(-5) == 0.0
105 +
106 + def test_low_stars(self):
107 + score = score_stars(10)
108 + assert 0 < score < 25
109 +
110 + def test_high_stars(self):
111 + score = score_stars(10000)
112 + assert score == 25.0
113 +
114 + def test_very_high_stars_capped(self):
115 + score = score_stars(1_000_000)
116 + assert score == 25.0
117 +
118 + def test_diminishing_returns(self):
119 + s10 = score_stars(10)
120 + s100 = score_stars(100)
121 + s1000 = score_stars(1000)
122 + # Log scale means equal absolute gains per 10x, but relative gains shrink
123 + assert s100 > s10
124 + assert s1000 > s100
125 + # Verify sublinear: doubling stars doesn't double score
126 + assert score_stars(200) < score_stars(100) * 2
127 +
128 +
129 +# --- Test score_stars_gained ---
130 +
131 +
132 +class TestScoreStarsGained:
133 + def test_zero_gained(self):
134 + assert score_stars_gained(0) == 0.0
135 +
136 + def test_negative_gained(self):
137 + assert score_stars_gained(-10) == 0.0
138 +
139 + def test_moderate_gained(self):
140 + score = score_stars_gained(50)
141 + assert 0 < score < 25
142 +
143 + def test_high_gained_capped(self):
144 + score = score_stars_gained(10000)
145 + assert score == 25.0
146 +
147 +
148 +# --- Test score_language ---
149 +
150 +
151 +class TestScoreLanguage:
152 + def test_no_language(self):
153 + assert score_language(None, {"Python": 1.2}) == 7.5
154 +
155 + def test_no_boost_config(self):
156 + assert score_language("Python", {}) == 7.5
157 +
158 + def test_matching_language_boost(self):
159 + score = score_language("Python", {"Python": 1.2})
160 + assert score == pytest.approx(9.0)
161 +
162 + def test_non_matching_language(self):
163 + score = score_language("Rust", {"Python": 1.2})
164 + assert score == 7.5
165 +
166 + def test_high_boost_capped(self):
167 + score = score_language("Python", {"Python": 3.0})
168 + assert score == 15.0
169 +
170 +
171 +# --- Test score_topics ---
172 +
173 +
174 +class TestScoreTopics:
175 + def test_no_repo_topics(self):
176 + assert score_topics([], ["machine-learning"]) == 0.0
177 +
178 + def test_no_relevance_list(self):
179 + assert score_topics(["python"], []) == 0.0
180 +
181 + def test_single_match(self):
182 + score = score_topics(["machine-learning"], ["machine-learning", "deep-learning", "llm"])
183 + assert score == pytest.approx(25.0 / 3)
184 +
185 + def test_full_match(self):
186 + topics = ["machine-learning", "deep-learning", "llm"]
187 + relevance = ["machine-learning", "deep-learning", "llm", "transformers"]
188 + score = score_topics(topics, relevance)
189 + assert score == 25.0 # 3 matches, capped at 3
190 +
191 + def test_case_insensitive(self):
192 + score = score_topics(["Machine-Learning"], ["machine-learning", "deep-learning", "llm"])
193 + assert score > 0
194 +
195 + def test_no_overlap(self):
196 + assert score_topics(["rust", "wasm"], ["machine-learning", "deep-learning", "llm"]) == 0.0
197 +
198 +
199 +# --- Test score_age ---
200 +
201 +
202 +class TestScoreAge:
203 + def test_no_created_at(self):
204 + assert score_age(None, 365) == 5.0
205 +
206 + def test_invalid_date(self):
207 + assert score_age("not-a-date", 365) == 5.0
208 +
209 + def test_brand_new_repo(self):
210 + now_iso = datetime.now(UTC).isoformat()
211 + score = score_age(now_iso, 365)
212 + assert score == pytest.approx(10.0, abs=0.1)
213 +
214 + def test_old_repo_at_max_age(self):
215 + old = (datetime.now(UTC) - timedelta(days=365)).isoformat()
216 + score = score_age(old, 365)
217 + assert score == pytest.approx(5.0, abs=0.1)
218 +
219 + def test_very_old_repo_penalized(self):
220 + ancient = (datetime.now(UTC) - timedelta(days=730)).isoformat()
221 + score = score_age(ancient, 365)
222 + assert score < 5.0
223 +
224 + def test_extremely_old_repo_zero(self):
225 + ancient = (datetime.now(UTC) - timedelta(days=1000)).isoformat()
226 + score = score_age(ancient, 365)
227 + assert score == pytest.approx(0.0, abs=0.5)
228 +
229 +
230 +# --- Test compute_relevance_score ---
231 +
232 +
233 +class TestComputeRelevanceScore:
234 + def test_high_quality_repo(self, sample_repo, scoring_config):
235 + score = compute_relevance_score(sample_repo, scoring_config)
236 + assert 60 <= score <= 100
237 +
238 + def test_low_quality_repo(self, scoring_config):
239 + repo = {
240 + "name": "old-thing",
241 + "stars": 5,
242 + "stars_gained": 0,
243 + "language": "Shell",
244 + "topics": [],
245 + "created_at": (datetime.now(UTC) - timedelta(days=800)).isoformat(),
246 + }
247 + score = compute_relevance_score(repo, scoring_config)
248 + assert score < 40
249 +
250 + def test_empty_repo(self, scoring_config):
251 + score = compute_relevance_score({}, scoring_config)
252 + assert 0 <= score <= 100
253 +
254 + def test_score_bounded(self, scoring_config):
255 + repo = {
256 + "stars": 1_000_000,
257 + "stars_gained": 100_000,
258 + "language": "Python",
259 + "topics": ["machine-learning", "deep-learning", "llm", "transformers"],
260 + "created_at": datetime.now(UTC).isoformat(),
261 + }
262 + score = compute_relevance_score(repo, scoring_config)
263 + assert score <= 100.0
264 +
265 +
266 +# --- Test score_repos ---
267 +
268 +
269 +class TestScoreRepos:
270 + def test_filters_below_threshold(self, scoring_config):
271 + repos = [
272 + {"name": "good", "stars": 500, "stars_gained": 100, "language": "Python",
273 + "topics": ["machine-learning", "deep-learning"], "created_at": datetime.now(UTC).isoformat()},
274 + {"name": "bad", "stars": 2, "stars_gained": 0, "language": "Shell",
275 + "topics": [], "created_at": (datetime.now(UTC) - timedelta(days=800)).isoformat()},
276 + ]
277 + scored = score_repos(repos, scoring_config)
278 + names = [r["name"] for r in scored]
279 + assert "good" in names
280 + assert "bad" not in names
281 +
282 + def test_sorted_descending(self, scoring_config):
283 + repos = [
284 + {"name": "medium", "stars": 100, "stars_gained": 20, "language": "Python",
285 + "topics": ["machine-learning"], "created_at": datetime.now(UTC).isoformat()},
286 + {"name": "high", "stars": 5000, "stars_gained": 500, "language": "Python",
287 + "topics": ["machine-learning", "deep-learning", "llm"], "created_at": datetime.now(UTC).isoformat()},
288 + ]
289 + scored = score_repos(repos, scoring_config)
290 + assert len(scored) >= 1
291 + if len(scored) >= 2:
292 + assert scored[0]["relevance_score"] >= scored[1]["relevance_score"]
293 +
294 + def test_adds_relevance_score_field(self, scoring_config):
295 + repos = [
296 + {"name": "test", "stars": 500, "stars_gained": 50, "language": "Python",
297 + "topics": ["machine-learning"], "created_at": datetime.now(UTC).isoformat()},
298 + ]
299 + scored = score_repos(repos, scoring_config)
300 + assert len(scored) > 0
301 + assert "relevance_score" in scored[0]
302 + assert isinstance(scored[0]["relevance_score"], float)
303 +
304 + def test_empty_input(self, scoring_config):
305 + assert score_repos([], scoring_config) == []
306 +
307 +
308 +# --- Test load_config ---
309 +
310 +
311 +class TestLoadConfig:
312 + def test_load_valid_config(self, config_file):
313 + config = load_config(config_file)
314 + assert "scoring" in config
315 + assert config["scoring"]["min_stars"] == 20
316 +
317 + def test_missing_file(self):
318 + with pytest.raises(FileNotFoundError):
319 + load_config("nonexistent.yml")
320 +
321 +
322 +# --- Test get_scoring_config ---
323 +
324 +
325 +class TestGetScoringConfig:
326 + def test_with_scoring_section(self):
327 + config = {"scoring": {"min_stars": 50, "language_boost": {"Go": 1.3}}}
328 + sc = get_scoring_config(config)
329 + assert sc["min_stars"] == 50
330 + assert sc["language_boost"] == {"Go": 1.3}
331 +
332 + def test_without_scoring_section(self):
333 + sc = get_scoring_config({})
334 + assert sc["min_stars"] == 20
335 + assert sc["min_relevance_score"] == 40
336 +
337 +
338 +# --- Test find_latest_raw_json ---
339 +
340 +
341 +class TestFindLatestRawJson:
342 + def test_no_directory(self):
343 + assert find_latest_raw_json("nonexistent-topic-xyz") is None
344 +
345 + def test_finds_latest(self, tmp_path, monkeypatch):
346 + topic_dir = tmp_path / "raw" / "test-topic"
347 + topic_dir.mkdir(parents=True)
348 + (topic_dir / "2026-W20.json").write_text("[]")
349 + (topic_dir / "2026-W21.json").write_text("[]")
350 +
351 + monkeypatch.setattr("scripts.score_repos.raw_dir", lambda tid: topic_dir)
352 + result = find_latest_raw_json("test-topic")
353 + assert result is not None
354 + assert "W21" in result.name
355 +
356 +
357 +# --- Test CLI (main) ---
358 +
359 +
360 +class TestMain:
361 + def test_with_input_file(self, tmp_path, config_file):
362 + repos = [
363 + {"name": "repo1", "stars": 500, "stars_gained": 100, "language": "Python",
364 + "topics": ["machine-learning", "deep-learning"], "created_at": datetime.now(UTC).isoformat()},
365 + ]
366 + input_file = tmp_path / "input.json"
367 + input_file.write_text(json.dumps(repos))
368 + output_file = tmp_path / "output.json"
369 +
370 + result = main(["--config", str(config_file), "--input", str(input_file),
371 + "--output", str(output_file)])
372 + assert result == 0
373 + scored = json.loads(output_file.read_text())
374 + assert len(scored) == 1
375 + assert "relevance_score" in scored[0]
376 +
377 + def test_missing_input_file(self, config_file):
378 + result = main(["--config", str(config_file), "--input", "no_such_file.json"])
379 + assert result == 1
380 +
381 + def test_invalid_json_content(self, tmp_path, config_file):
382 + input_file = tmp_path / "bad.json"
383 + input_file.write_text('{"not": "a list"}')
384 + result = main(["--config", str(config_file), "--input", str(input_file)])
385 + assert result == 1
386 +
387 + def test_stdout_output(self, tmp_path, config_file, capsys):
388 + repos = [
389 + {"name": "repo1", "stars": 1000, "stars_gained": 200, "language": "Python",
390 + "topics": ["machine-learning", "llm"], "created_at": datetime.now(UTC).isoformat()},
391 + ]
392 + input_file = tmp_path / "input.json"
393 + input_file.write_text(json.dumps(repos))
394 +
395 + result = main(["--config", str(config_file), "--input", str(input_file)])
396 + assert result == 0
397 + output = capsys.readouterr().out
398 + parsed = json.loads(output)
399 + assert len(parsed) >= 1
400 +
401 + def test_no_input_no_raw_files(self, config_file):
402 + result = main(["--config", str(config_file), "--topic", "nonexistent-xyz-topic"])
403 + assert result == 1