feat: crawler reads queries from topic config (#62) (#98)
Juan Manuel Servera committed
May 19, 2026 at 15:49 UTC
5a908ff67711bf6c4f673eed4786ec50cc797275
4 files changed
+244
-13
scripts/crawl.py
+66
-11
@@ -484,9 +484,48 @@ def parse_args() -> argparse.Namespace:
484
default=None,
485
help="Topic ID for namespaced data directories. Defaults to 'general' (flat layout).",
486
)
487
+ parser.add_argument(
488
+ "--config",
489
+ default=None,
490
+ help="Path to a topic YAML config file (e.g. squadscope.topic.yml). "
491
+ "When provided, queries are read from the config instead of using hardcoded defaults.",
492
+ )
493
return parser.parse_args()
494
495
496
+def load_topic_queries(config_path: str, template_vars: dict[str, str]) -> dict[str, Any]:
497
+ """Load and resolve queries from a topic YAML config file.
498
+
499
+ Returns a dict with keys: primary (list[str]), secondary (list[str]), min_repos_per_week (int).
500
+ Template variables in queries (e.g. {last_week}, {today}) are replaced with values from template_vars.
501
+ """
502
+ import yaml
503
+
504
+ path = Path(config_path)
505
+ if not path.exists():
506
+ raise FileNotFoundError(f"Topic config not found: {config_path}")
507
+
508
+ with open(path, encoding="utf-8") as f:
509
+ data = yaml.safe_load(f)
510
+
511
+ queries_section = data.get("queries", {})
512
+ primary = queries_section.get("primary", [])
513
+ secondary = queries_section.get("secondary", [])
514
+ quality_section = data.get("quality", {})
515
+ min_repos = quality_section.get("min_repos_per_week", 5)
516
+
517
+ def resolve(q: str) -> str:
518
+ for key, value in template_vars.items():
519
+ q = q.replace(f"{{{key}}}", value)
520
+ return q
521
+
522
+ return {
523
+ "primary": [resolve(q) for q in primary],
524
+ "secondary": [resolve(q) for q in secondary],
525
+ "min_repos_per_week": min_repos,
526
+ }
527
+
528
+
529
def utc_now() -> datetime:
530
return datetime.now(UTC).replace(microsecond=0)
531
@@ -765,18 +804,34 @@ def main() -> int:
804
client = GitHubClient(github_token, cache_dir=topic_cache)
805
max_results = max(1, min(args.max_results, 1000))
806
768
- if args.as_of:
769
- created_filter = f"created:{since.date().isoformat()}..{window_end.date().isoformat()}"
770
- pushed_filter = f"pushed:{since.date().isoformat()}..{window_end.date().isoformat()}"
807
+ if args.config:
808
+ template_vars = {
809
+ "last_week": since.date().isoformat(),
810
+ "today": window_end.date().isoformat(),
811
+ }
812
+ topic_queries = load_topic_queries(args.config, template_vars)
813
+ all_candidates: list[Any] = []
814
+ for q in topic_queries["primary"]:
815
+ all_candidates.extend(client.search_repositories(q, max_results=max_results))
816
+ if len(all_candidates) < topic_queries["min_repos_per_week"]:
817
+ for q in topic_queries["secondary"]:
818
+ all_candidates.extend(client.search_repositories(q, max_results=max_results))
819
+ new_candidates = all_candidates
820
+ previous_stars = load_previous_star_snapshot(SNAPSHOT_ROOT, week, output_path.parent, RAW_ROOT)
821
+ trending_candidates: list[Any] = []
822
else:
772
- created_filter = f"created:>{since.date().isoformat()}"
773
- pushed_filter = f"pushed:>{since.date().isoformat()}"
774
- new_query = f"{created_filter} stars:>50"
775
- trending_query = f"{pushed_filter} stars:>50"
776
-
777
- new_candidates = client.search_repositories(new_query, max_results=max_results)
778
- previous_stars = load_previous_star_snapshot(SNAPSHOT_ROOT, week, output_path.parent, RAW_ROOT)
779
- trending_candidates = client.search_repositories(trending_query, max_results=max_results)
823
+ if args.as_of:
824
+ created_filter = f"created:{since.date().isoformat()}..{window_end.date().isoformat()}"
825
+ pushed_filter = f"pushed:{since.date().isoformat()}..{window_end.date().isoformat()}"
826
+ else:
827
+ created_filter = f"created:>{since.date().isoformat()}"
828
+ pushed_filter = f"pushed:>{since.date().isoformat()}"
829
+ new_query = f"{created_filter} stars:>50"
830
+ trending_query = f"{pushed_filter} stars:>50"
831
+
832
+ new_candidates = client.search_repositories(new_query, max_results=max_results)
833
+ previous_stars = load_previous_star_snapshot(SNAPSHOT_ROOT, week, output_path.parent, RAW_ROOT)
834
+ trending_candidates = client.search_repositories(trending_query, max_results=max_results)
835
836
new_repos, new_filters = collect_repositories(client, new_candidates)
837
trending_repos, trending_filters = collect_repositories(
tests/test_crawl.py
+2
-2
@@ -118,7 +118,7 @@ class CrawlTests(unittest.TestCase):
118
def has_readme(self, full_name: str) -> bool:
119
return True
120
121
- args = Namespace(since="2026-05-11", as_of=None, max_results=25, output="data/raw/test-live.json", topic=None)
121
+ args = Namespace(since="2026-05-11", as_of=None, max_results=25, output="data/raw/test-live.json", topic=None, config=None)
122
with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(
123
"os.environ", {"GITHUB_TOKEN": "token"}, clear=False
124
), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(
@@ -151,7 +151,7 @@ class CrawlTests(unittest.TestCase):
151
def has_readme(self, full_name: str) -> bool:
152
return True
153
154
- args = Namespace(since="2026-05-11", as_of="2026-05-18", max_results=25, output="data/raw/test-backfill.json", topic=None)
154
+ args = Namespace(since="2026-05-11", as_of="2026-05-18", max_results=25, output="data/raw/test-backfill.json", topic=None, config=None)
155
with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(
156
"os.environ", {"GITHUB_TOKEN": "token"}, clear=False
157
), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(
tests/test_crawl_topic_queries.py
new
+175
@@ -0,0 +1,175 @@
1
+"""Tests for topic-config-driven query loading in crawl.py."""
2
+
3
+import os
4
+import tempfile
5
+import unittest
6
+from argparse import Namespace
7
+from pathlib import Path
8
+from unittest import mock
9
+
10
+import scripts.crawl as crawl
11
+
12
+
13
+SAMPLE_CONFIG = """\
14
+topic:
15
+ id: ai-ml
16
+ name: "AI & Machine Learning"
17
+
18
+queries:
19
+ primary:
20
+ - "topic:machine-learning stars:>50 pushed:>{last_week}"
21
+ - "topic:artificial-intelligence stars:>50 pushed:>{last_week}"
22
+ secondary:
23
+ - "topic:transformers stars:>100 pushed:>{last_week}"
24
+
25
+quality:
26
+ min_repos_per_week: 5
27
+"""
28
+
29
+TESTS_ROOT = Path(__file__).resolve().parent
30
+
31
+
32
+class LoadTopicQueriesTests(unittest.TestCase):
33
+ def test_resolves_template_variables(self) -> None:
34
+ with tempfile.NamedTemporaryFile(
35
+ mode="w", suffix=".yml", dir=TESTS_ROOT, delete=False
36
+ ) as f:
37
+ f.write(SAMPLE_CONFIG)
38
+ f.flush()
39
+ config_path = f.name
40
+
41
+ try:
42
+ result = crawl.load_topic_queries(
43
+ config_path, {"last_week": "2026-05-11", "today": "2026-05-18"}
44
+ )
45
+ self.assertEqual(
46
+ result["primary"],
47
+ [
48
+ "topic:machine-learning stars:>50 pushed:>2026-05-11",
49
+ "topic:artificial-intelligence stars:>50 pushed:>2026-05-11",
50
+ ],
51
+ )
52
+ self.assertEqual(
53
+ result["secondary"],
54
+ ["topic:transformers stars:>100 pushed:>2026-05-11"],
55
+ )
56
+ self.assertEqual(result["min_repos_per_week"], 5)
57
+ finally:
58
+ os.unlink(config_path)
59
+
60
+ def test_raises_on_missing_file(self) -> None:
61
+ with self.assertRaises(FileNotFoundError):
62
+ crawl.load_topic_queries("/nonexistent.yml", {})
63
+
64
+
65
+class MainWithConfigTests(unittest.TestCase):
66
+ def _make_fake_client_class(self, queries: list[str], results_per_query: int = 0):
67
+ """Return a FakeClient class that records queries and returns N fake repos."""
68
+
69
+ class FakeClient:
70
+ def __init__(self, token: str, **kwargs) -> None:
71
+ self.token = token
72
+ self.api_calls_used = 0
73
+ self.cache_hits = 0
74
+ self.stale_cache_hits = 0
75
+ self.rate_limit_limit = None
76
+ self.rate_limit_remaining = None
77
+ self.rate_limit_reset = None
78
+ self.rate_limit_resource = None
79
+ self.errors = []
80
+
81
+ def search_repositories(self, query: str, *, max_results: int = 1000):
82
+ queries.append(query)
83
+ return [{"full_name": f"org/repo-{i}", "stargazers_count": 100}
84
+ for i in range(results_per_query)]
85
+
86
+ def has_readme(self, full_name: str) -> bool:
87
+ return True
88
+
89
+ return FakeClient
90
+
91
+ def test_config_uses_primary_queries_only_when_enough_repos(self) -> None:
92
+ queries: list[str] = []
93
+ with tempfile.NamedTemporaryFile(
94
+ mode="w", suffix=".yml", dir=TESTS_ROOT, delete=False
95
+ ) as f:
96
+ f.write(SAMPLE_CONFIG)
97
+ f.flush()
98
+ config_path = f.name
99
+
100
+ try:
101
+ FakeClient = self._make_fake_client_class(queries, results_per_query=5)
102
+ args = Namespace(
103
+ since="2026-05-11", as_of=None, max_results=25,
104
+ output="data/raw/test-config.json", topic=None, config=config_path,
105
+ )
106
+ with mock.patch.object(crawl, "parse_args", return_value=args), \
107
+ mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False), \
108
+ mock.patch.object(crawl, "GitHubClient", FakeClient), \
109
+ mock.patch.object(crawl, "load_previous_star_snapshot", return_value={}), \
110
+ mock.patch.object(crawl, "write_payload"), \
111
+ mock.patch.object(crawl, "print"):
112
+ exit_code = crawl.main()
113
+
114
+ self.assertEqual(exit_code, 0)
115
+ # Only primary queries run (2 primaries), secondary not needed
116
+ self.assertEqual(len(queries), 2)
117
+ self.assertIn("topic:machine-learning", queries[0])
118
+ self.assertIn("topic:artificial-intelligence", queries[1])
119
+ self.assertIn("pushed:>2026-05-11", queries[0])
120
+ finally:
121
+ os.unlink(config_path)
122
+
123
+ def test_config_runs_secondary_when_primary_insufficient(self) -> None:
124
+ queries: list[str] = []
125
+ with tempfile.NamedTemporaryFile(
126
+ mode="w", suffix=".yml", dir=TESTS_ROOT, delete=False
127
+ ) as f:
128
+ f.write(SAMPLE_CONFIG)
129
+ f.flush()
130
+ config_path = f.name
131
+
132
+ try:
133
+ # Return only 1 repo per query → 2 total from primaries < min_repos_per_week=5
134
+ FakeClient = self._make_fake_client_class(queries, results_per_query=1)
135
+ args = Namespace(
136
+ since="2026-05-11", as_of=None, max_results=25,
137
+ output="data/raw/test-config-secondary.json", topic=None, config=config_path,
138
+ )
139
+ with mock.patch.object(crawl, "parse_args", return_value=args), \
140
+ mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False), \
141
+ mock.patch.object(crawl, "GitHubClient", FakeClient), \
142
+ mock.patch.object(crawl, "load_previous_star_snapshot", return_value={}), \
143
+ mock.patch.object(crawl, "write_payload"), \
144
+ mock.patch.object(crawl, "print"):
145
+ exit_code = crawl.main()
146
+
147
+ self.assertEqual(exit_code, 0)
148
+ # 2 primary + 1 secondary = 3 queries
149
+ self.assertEqual(len(queries), 3)
150
+ self.assertIn("topic:transformers", queries[2])
151
+ finally:
152
+ os.unlink(config_path)
153
+
154
+ def test_no_config_preserves_existing_behavior(self) -> None:
155
+ """Without --config, queries use the hardcoded stars:>50 pattern."""
156
+ queries: list[str] = []
157
+ FakeClient = self._make_fake_client_class(queries, results_per_query=0)
158
+ args = Namespace(
159
+ since="2026-05-11", as_of=None, max_results=25,
160
+ output="data/raw/test-noconfig.json", topic=None, config=None,
161
+ )
162
+ with mock.patch.object(crawl, "parse_args", return_value=args), \
163
+ mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False), \
164
+ mock.patch.object(crawl, "GitHubClient", FakeClient), \
165
+ mock.patch.object(crawl, "load_previous_star_snapshot", return_value={}), \
166
+ mock.patch.object(crawl, "write_payload"), \
167
+ mock.patch.object(crawl, "print"):
168
+ exit_code = crawl.main()
169
+
170
+ self.assertEqual(exit_code, 0)
171
+ self.assertEqual(queries, ["created:>2026-05-11 stars:>50", "pushed:>2026-05-11 stars:>50"])
172
+
173
+
174
+if __name__ == "__main__":
175
+ unittest.main()
tests/test_pipeline.py
+1
@@ -315,6 +315,7 @@ class PipelineIntegrationTests(unittest.TestCase):
315
max_results=10,
316
output=str(output_path),
317
topic=None,
318
+ config=None,
319
)
320
321
with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(