@cryptotaxi247 / SquadScope-1
Code Commits Issues Pulls Sessions CI/CD
main
main
py 194 lines 6.97 KB
Raw
1 """Tests for topic-config-driven query loading in crawl.py."""
2
3 import os
4 import tempfile
5 import unittest
6 from argparse import Namespace
7 from pathlib import Path
8 from unittest import mock
9
10 import scripts.crawl as crawl
11
12 SAMPLE_CONFIG = """\
13 topic:
14 id: ai-ml
15 name: "AI & Machine Learning"
16
17 queries:
18 primary:
19 - "topic:machine-learning stars:>50 pushed:>{last_week}"
20 - "topic:artificial-intelligence stars:>50 pushed:>{last_week}"
21 secondary:
22 - "topic:transformers stars:>100 pushed:>{last_week}"
23
24 quality:
25 min_repos_per_week: 5
26 """
27
28 TESTS_ROOT = Path(__file__).resolve().parent
29
30
31 class LoadTopicQueriesTests(unittest.TestCase):
32 def test_resolves_template_variables(self) -> None:
33 with tempfile.NamedTemporaryFile(
34 mode="w", suffix=".yml", dir=TESTS_ROOT, delete=False
35 ) as f:
36 f.write(SAMPLE_CONFIG)
37 f.flush()
38 config_path = f.name
39
40 try:
41 result = crawl.load_topic_queries(
42 config_path, {"last_week": "2026-05-11", "today": "2026-05-18"}
43 )
44 self.assertEqual(
45 result["primary"],
46 [
47 "topic:machine-learning stars:>50 pushed:>2026-05-11",
48 "topic:artificial-intelligence stars:>50 pushed:>2026-05-11",
49 ],
50 )
51 self.assertEqual(
52 result["secondary"],
53 ["topic:transformers stars:>100 pushed:>2026-05-11"],
54 )
55 self.assertEqual(result["min_repos_per_week"], 5)
56 finally:
57 os.unlink(config_path)
58
59 def test_raises_on_missing_file(self) -> None:
60 with self.assertRaises(FileNotFoundError):
61 crawl.load_topic_queries("/nonexistent.yml", {})
62
63
64 class MainWithConfigTests(unittest.TestCase):
65 def _make_fake_client_class(self, queries: list[str], results_per_query: int = 0):
66 """Return a FakeClient class that records queries and returns N fake repos."""
67
68 class FakeClient:
69 def __init__(self, token: str, **kwargs) -> None:
70 self.token = token
71 self.api_calls_used = 0
72 self.cache_hits = 0
73 self.stale_cache_hits = 0
74 self.rate_limit_limit = None
75 self.rate_limit_remaining = None
76 self.rate_limit_reset = None
77 self.rate_limit_resource = None
78 self.errors = []
79
80 def search_repositories(self, query: str, *, max_results: int = 1000):
81 queries.append(query)
82 return [
83 {"full_name": f"org/repo-{i}", "stargazers_count": 100}
84 for i in range(results_per_query)
85 ]
86
87 def has_readme(self, full_name: str) -> bool:
88 return True
89
90 return FakeClient
91
92 def test_config_uses_primary_queries_only_when_enough_repos(self) -> None:
93 queries: list[str] = []
94 with tempfile.NamedTemporaryFile(
95 mode="w", suffix=".yml", dir=TESTS_ROOT, delete=False
96 ) as f:
97 f.write(SAMPLE_CONFIG)
98 f.flush()
99 config_path = f.name
100
101 try:
102 FakeClient = self._make_fake_client_class(queries, results_per_query=5)
103 args = Namespace(
104 since="2026-05-11",
105 as_of=None,
106 max_results=25,
107 output="data/raw/test-config.json",
108 topic=None,
109 config=config_path,
110 )
111 with (
112 mock.patch.object(crawl, "parse_args", return_value=args),
113 mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False),
114 mock.patch.object(crawl, "GitHubClient", FakeClient),
115 mock.patch.object(crawl, "load_previous_star_snapshot", return_value={}),
116 mock.patch.object(crawl, "write_payload"),
117 mock.patch.object(crawl, "print"),
118 ):
119 exit_code = crawl.main()
120
121 self.assertEqual(exit_code, 0)
122 # Only primary queries run (2 primaries), secondary not needed
123 self.assertEqual(len(queries), 2)
124 self.assertIn("topic:machine-learning", queries[0])
125 self.assertIn("topic:artificial-intelligence", queries[1])
126 self.assertIn("pushed:>2026-05-11", queries[0])
127 finally:
128 os.unlink(config_path)
129
130 def test_config_runs_secondary_when_primary_insufficient(self) -> None:
131 queries: list[str] = []
132 with tempfile.NamedTemporaryFile(
133 mode="w", suffix=".yml", dir=TESTS_ROOT, delete=False
134 ) as f:
135 f.write(SAMPLE_CONFIG)
136 f.flush()
137 config_path = f.name
138
139 try:
140 # Return only 1 repo per query → 2 total from primaries < min_repos_per_week=5
141 FakeClient = self._make_fake_client_class(queries, results_per_query=1)
142 args = Namespace(
143 since="2026-05-11",
144 as_of=None,
145 max_results=25,
146 output="data/raw/test-config-secondary.json",
147 topic=None,
148 config=config_path,
149 )
150 with (
151 mock.patch.object(crawl, "parse_args", return_value=args),
152 mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False),
153 mock.patch.object(crawl, "GitHubClient", FakeClient),
154 mock.patch.object(crawl, "load_previous_star_snapshot", return_value={}),
155 mock.patch.object(crawl, "write_payload"),
156 mock.patch.object(crawl, "print"),
157 ):
158 exit_code = crawl.main()
159
160 self.assertEqual(exit_code, 0)
161 # 2 primary + 1 secondary = 3 queries
162 self.assertEqual(len(queries), 3)
163 self.assertIn("topic:transformers", queries[2])
164 finally:
165 os.unlink(config_path)
166
167 def test_no_config_preserves_existing_behavior(self) -> None:
168 """Without --config, queries use the hardcoded stars:>50 pattern."""
169 queries: list[str] = []
170 FakeClient = self._make_fake_client_class(queries, results_per_query=0)
171 args = Namespace(
172 since="2026-05-11",
173 as_of=None,
174 max_results=25,
175 output="data/raw/test-noconfig.json",
176 topic=None,
177 config=None,
178 )
179 with (
180 mock.patch.object(crawl, "parse_args", return_value=args),
181 mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False),
182 mock.patch.object(crawl, "GitHubClient", FakeClient),
183 mock.patch.object(crawl, "load_previous_star_snapshot", return_value={}),
184 mock.patch.object(crawl, "write_payload"),
185 mock.patch.object(crawl, "print"),
186 ):
187 exit_code = crawl.main()
188
189 self.assertEqual(exit_code, 0)
190 self.assertEqual(queries, ["created:>2026-05-11 stars:>50", "pushed:>2026-05-11 stars:>50"])
191
192
193 if __name__ == "__main__":
194 unittest.main()