@cryptotaxi247 / SquadScope / commits / 779f9ef

Fix crawler review findings

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

jmservera committed May 18, 2026 at 12:18 UTC 779f9ef288f26517b120d84ebf8c257dc945761d

4 files changed +289 -43

.squad/agents/bender/history.md

+1 -2

@@ -14,8 +14,7 @@

15 ## Learnings

17 +- **2026-05-18T12:07:20.778+02:00:** Copilot review follow-up on crawler hardening: keep star snapshots broad for `stars_gained`, but document that they intentionally cover pre-filter candidates; restore `get_json()` payload compatibility via an internal `get_json_entry()` helper; treat malformed JSON as non-retryable; and search both `RAW_ROOT` and custom `--output` parents when loading prior star snapshots so reruns keep working.

18 - **2026-05-18T10:06:38.734+02:00:** GitHub Actions can run the standalone `copilot` CLI (`@github/copilot`) in programmatic mode with `copilot -p ...`. The safest documented CI auth flow is a fine-grained PAT with the **Copilot Requests** account permission passed as `COPILOT_GITHUB_TOKEN`; `gh auth token` only exposes an existing `gh` token and `gh-copilot` is deprecated in favor of the standalone CLI. GitHub Models (`models: read`) is the clean fallback if direct Copilot CLI automation proves brittle.

19 - **2026-05-18T10:11:20Z:** Team decided Phase 0 PRD decomposition is final; 24 GitHub issues created (4 investigation + 20 implementation). PRD decomposition captures all decisions. MCP tools can crawl beyond GitHub with remote call allowlist. Ready for issue creation.

20 - **2026-05-18T10:27:35.339+02:00:** The crawler now uses `GET /search/repositories` for both `created:>{last_week_date} stars:>50` and `pushed:>{last_week_date} stars:>50`, comparing current stars against the most recent prior `data/raw/*.json` snapshot when available to estimate weekly star gains. It authenticates with `GITHUB_TOKEN`, paginates up to the GitHub Search API's 1,000-result ceiling, caches README checks in-process, applies exponential backoff with jitter for rate limits, and skips repos whose README lookup is blocked by org SAML enforcement.

20 -- **2026-05-18T10:59:10.800+02:00:** Hardening issue #6 required persistent `data/cache/` response caching, dedicated `data/snapshots/YYYY-WNN-stars.json` star maps, stronger low-signal heuristics (fork/template/tutorial/homework filters), and bounded README retries so a real `GITHUB_TOKEN=$(gh auth token) python3 scripts/crawl.py` run can complete while preserving partial data and logging rate-limit state.

21 -- **2026-05-18T10:59:10Z:** Issue #5 complete. Commit fb14275 (209 new repos, 215 trending in data/raw/2026-W21.json). Ready for Issue #6+. User directive: all future work follows branch → PR → Review → Merge workflow (no direct commits to main).

README.md

@@ -28,6 +28,12 @@ SquadScope is a Hugo-powered GitHub Pages site for weekly, monthly, and yearly t

28 - `data/analyzed/` — analysis output

29 - `data/snapshots/` — star count snapshots

31 +## Crawler notes

32 +

33 +- `signals.top_topics` de-duplicates repositories by `full_name` across the new and trending buckets before counting topics, so a repo found by both searches only contributes once.

34 +- `data/snapshots/YYYY-WNN-stars.json` intentionally stores the broader pre-filter search candidate universe to preserve week-over-week `stars_gained` comparisons even when a repo is later filtered out of the published payload.

35 +- Live runs use open-ended `created:>` / `pushed:>` GitHub search filters; `--as-of` runs switch to bounded date ranges so historical backfills stay deterministic.

36 +

37 ## Deployment

39 Pushing to `main` triggers `.github/workflows/deploy-site.yml`, which builds the Hugo site and deploys the generated `public/` directory to GitHub Pages.

scripts/crawl.py

+109 -41

@@ -9,7 +9,6 @@ import json

9 import os

10 import random

11 import re

12 -import socket

12 import sys

13 import time

14 from collections import Counter

@@ -35,10 +34,7 @@ LOW_SIGNAL_TOPICS = {

34 "examples",

35 "exercise",

36 "homework",

38 - "lab",

37 "learning",

40 - "practice",

41 - "starter",

38 "starter-template",

39 "template",

40 "templates",

@@ -54,26 +50,28 @@ LOW_SIGNAL_TOKENS = {

50 "cheatsheet",

51 "course",

52 "courses",

57 - "demo",

53 "example",

54 "examples",

55 "exercise",

56 "exercises",

57 "homework",

58 + "leetcode",

59 + "template",

60 + "tutorial",

61 + "tutorials",

62 + "walkthrough",

63 + "workshop",

64 +}

65 +LOW_SIGNAL_NAME_TOKENS = {

66 + "demo",

67 "kata",

68 "lab",

69 "labs",

66 - "leetcode",

70 "lesson",

71 "lessons",

72 "practice",

73 "sample",

74 "starter",

72 - "template",

73 - "tutorial",

74 - "tutorials",

75 - "walkthrough",

76 - "workshop",

75 }

76 LOW_SIGNAL_PHRASES = {

77 "course project",

@@ -138,7 +136,7 @@ class ResponseCache:

136 "headers": headers,

137 "payload": payload,

138 }

141 - path.write_text(json.dumps(cache_payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")

139 + path.write_text(json.dumps(cache_payload, separators=(",", ":"), ensure_ascii=False) + "\n", encoding="utf-8")

140

141 def _path_for(self, key: str) -> Path:

142 parsed = parse.urlparse(key)

@@ -179,6 +177,27 @@ class GitHubClient:

177 allow_stale: bool = True,

178 max_retries: int | None = None,

179 max_delay_seconds: float = 300.0,

180 + ) -> Any:

181 + return self.get_json_entry(

182 + url,

183 + params,

184 + acceptable_statuses=acceptable_statuses,

185 + ttl_seconds=ttl_seconds,

186 + allow_stale=allow_stale,

187 + max_retries=max_retries,

188 + max_delay_seconds=max_delay_seconds,

189 + ).payload

190 +

191 + def get_json_entry(

192 + self,

193 + url: str,

194 + params: dict[str, Any] | None = None,

195 + *,

196 + acceptable_statuses: set[int] | None = None,

197 + ttl_seconds: int | None = None,

198 + allow_stale: bool = True,

199 + max_retries: int | None = None,

200 + max_delay_seconds: float = 300.0,

201 ) -> CacheEntry:

202 query = f"{url}?{parse.urlencode(params)}" if params else url

203 accepted = acceptable_statuses or set()

@@ -209,8 +228,16 @@ class GitHubClient:

228 self.api_calls_used += 1

229 headers = {name: value for name, value in response.headers.items()}

230 self._update_rate_limit(headers)

212 - payload = json.loads(response.read().decode("utf-8", errors="replace"))

231 + body = response.read().decode("utf-8", errors="replace")

232 self._last_request_at = time.monotonic()

233 + try:

234 + payload = json.loads(body)

235 + except json.JSONDecodeError as exc:

236 + if stale_fallback is not None:

237 + self.stale_cache_hits += 1

238 + log(f"Using stale cache for {query} after malformed JSON response: {exc}")

239 + return stale_fallback

240 + raise RuntimeError(f"GitHub API returned malformed JSON for {query}: {exc}") from exc

241 self._cache.store(query, status=response.status, payload=payload, headers=self._cache_headers(headers))

242 self._log_rate_limit(query)

243 return CacheEntry(response.status, payload, headers, utc_now())

@@ -235,7 +262,7 @@ class GitHubClient:

262 ) from exc

263 self._sleep_before_retry(attempt, headers, body, query, retry_limit, max_delay_seconds)

264 attempt += 1

238 - except (error.URLError, TimeoutError, socket.timeout, json.JSONDecodeError) as exc:

265 + except (error.URLError, TimeoutError) as exc:

266 if attempt >= retry_limit:

267 if stale_fallback is not None:

268 self.stale_cache_hits += 1

@@ -251,7 +278,7 @@ class GitHubClient:

278 max_pages = min((max_results + per_page - 1) // per_page, 10)

279 for page in range(1, max_pages + 1):

280 try:

254 - response = self.get_json(

281 + response = self.get_json_entry(

282 SEARCH_REPOSITORIES,

283 params={

284 "q": query,

@@ -283,7 +310,7 @@ class GitHubClient:

310 return self._readme_cache[full_name]

311 url = f"{API_ROOT}/repos/{full_name}/readme"

312 try:

286 - response = self.get_json(

313 + response = self.get_json_entry(

314 url,

315 acceptable_statuses={404},

316 ttl_seconds=24 * 60 * 60,

@@ -331,6 +358,8 @@ class GitHubClient:

358 retry_after = max(float(headers["Retry-After"]), 1.0)

359 except ValueError:

360 retry_after = None

361 + if retry_after is not None and (self.rate_limit_reset is None or (self.rate_limit_remaining or 0) <= 0):

362 + self.rate_limit_reset = max(self.rate_limit_reset or 0, int(time.time() + retry_after))

363 base_delay = min(2**attempt, 60)

364 jitter = random.uniform(0.3, 1.7)

365 delay = retry_after or reset_delay or (base_delay + jitter)

@@ -355,8 +384,16 @@ class GitHubClient:

384 critical_threshold = max(3, min(10, int(self.rate_limit_limit * 0.03)))

385 if self.rate_limit_remaining > low_threshold:

386 return

358 - reset_delay = self._reset_delay({"X-RateLimit-Reset": str(self.rate_limit_reset)} if self.rate_limit_reset else None)

387 + reset_headers = {"X-RateLimit-Reset": str(self.rate_limit_reset)} if self.rate_limit_reset is not None else None

388 + reset_delay = self._reset_delay(reset_headers)

389 if reset_delay is None:

390 + if self.rate_limit_remaining <= critical_threshold:

391 + delay = 10.0 if self.rate_limit_remaining <= 0 else 3.0

392 + log(

393 + f"Rate limit low ({self.rate_limit_remaining}/{self.rate_limit_limit} {self.rate_limit_resource or 'requests'}) "

394 + f"without reset hint; cooling down {delay:.1f}s before {query}."

395 + )

396 + time.sleep(delay)

397 return

398 if self.rate_limit_remaining <= critical_threshold:

399 delay = min(reset_delay + random.uniform(0.3, 1.5), 300.0)

@@ -469,30 +506,46 @@ def decode_json_body(body: str) -> Any:

506 return {"message": body.strip()}

507

508

472 -def load_previous_star_snapshot(snapshot_dir: Path, raw_dir: Path, current_week: str) -> dict[str, int]:

509 +def load_previous_star_snapshot(snapshot_dir: Path, current_week: str, *raw_dirs: Path) -> dict[str, int]:

510 for snapshot in sorted(snapshot_dir.glob("*-stars.json"), reverse=True):

474 - stars = load_star_mapping(snapshot, current_week)

475 - if stars:

476 - return stars

477 - for snapshot in sorted(raw_dir.glob("*.json"), reverse=True):

478 - stars = load_star_mapping(snapshot, current_week)

511 + stars, reason = load_star_mapping_details(snapshot, current_week)

512 if stars:

513 return stars

514 + if reason and reason != "same-week snapshot":

515 + log(f"Skipping star snapshot {snapshot}: {reason}.")

516 + seen_dirs: set[Path] = set()

517 + for raw_dir in raw_dirs:

518 + if raw_dir in seen_dirs:

519 + continue

520 + seen_dirs.add(raw_dir)

521 + for snapshot in sorted(raw_dir.glob("*.json"), reverse=True):

522 + stars, reason = load_star_mapping_details(snapshot, current_week)

523 + if stars:

524 + return stars

525 + if reason and reason != "same-week snapshot":

526 + log(f"Skipping star snapshot {snapshot}: {reason}.")

527 return {}

528

529

530 def load_star_mapping(path: Path, current_week: str) -> dict[str, int]:

531 + return load_star_mapping_details(path, current_week)[0]

532 +

533 +

534 +def load_star_mapping_details(path: Path, current_week: str) -> tuple[dict[str, int], str | None]:

535 try:

536 payload = json.loads(path.read_text(encoding="utf-8"))

487 - except (OSError, json.JSONDecodeError):

488 - return {}

537 + except OSError as exc:

538 + return {}, f"read failed ({exc})"

539 + except json.JSONDecodeError as exc:

540 + return {}, f"invalid JSON ({exc})"

541 if payload.get("week") == current_week:

490 - return {}

542 + return {}, "same-week snapshot"

543 stars = payload.get("stars")

544 if isinstance(stars, dict):

545 mapping = {name: int(value) for name, value in stars.items() if isinstance(value, int)}

546 if mapping:

495 - return mapping

547 + return mapping, None

548 + return {}, "empty stars mapping"

549 mapping: dict[str, int] = {}

550 for section in ("new_repos", "trending_repos"):

551 for repo in payload.get(section, []):

@@ -500,7 +553,9 @@ def load_star_mapping(path: Path, current_week: str) -> dict[str, int]:

553 stars_value = repo.get("stars")

554 if full_name and isinstance(stars_value, int):

555 mapping[full_name] = stars_value

503 - return mapping

556 + if mapping:

557 + return mapping, None

558 + return {}, "no star data"

559

560

561 def tokenize(text: str) -> set[str]:

@@ -518,16 +573,13 @@ def significance_skip_reason(repo: dict[str, Any]) -> str | None:

573 topics = {str(topic).lower() for topic in repo.get("topics") or []}

574 if topics & LOW_SIGNAL_TOPICS:

575 return "low_signal_topic"

521 - combined_text = " ".join(

522 - [

523 - str(repo.get("name") or ""),

524 - description,

525 - " ".join(sorted(topics)),

526 - ]

527 - ).lower()

576 + name = str(repo.get("name") or "")

577 + combined_text = " ".join([name, description]).lower()

578 combined_tokens = tokenize(combined_text)

579 if combined_tokens & LOW_SIGNAL_TOKENS:

580 return "low_signal_keyword"

581 + if tokenize(name) & LOW_SIGNAL_NAME_TOKENS:

582 + return "low_signal_keyword"

583 if any(phrase in combined_text for phrase in LOW_SIGNAL_PHRASES):

584 return "low_signal_phrase"

585 return None

@@ -607,6 +659,7 @@ def collect_repositories(

659

660 def build_signals(*repo_groups: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:

661 topic_counter: Counter[str] = Counter()

662 + # A repo can appear in both search buckets; count its topics once to avoid inflating weekly signals.

663 merged: dict[str, dict[str, Any]] = {}

664 for group in repo_groups:

665 for repo in group:

@@ -662,10 +715,18 @@ def validate_payload(payload: dict[str, Any]) -> None:

715 raise ValueError("metadata.api_calls_used must be an integer")

716 if not isinstance(metadata.get("cache_hits"), int):

717 raise ValueError("metadata.cache_hits must be an integer")

718 + if not isinstance(metadata.get("stale_cache_hits"), int):

719 + raise ValueError("metadata.stale_cache_hits must be an integer")

720 if metadata.get("rate_limit_remaining") is not None and not isinstance(metadata.get("rate_limit_remaining"), int):

721 raise ValueError("metadata.rate_limit_remaining must be an integer or null")

722 if metadata.get("rate_limit_limit") is not None and not isinstance(metadata.get("rate_limit_limit"), int):

723 raise ValueError("metadata.rate_limit_limit must be an integer or null")

724 + if metadata.get("rate_limit_reset") is not None and not isinstance(metadata.get("rate_limit_reset"), int):

725 + raise ValueError("metadata.rate_limit_reset must be an integer or null")

726 + if metadata.get("rate_limit_resource") is not None and not isinstance(metadata.get("rate_limit_resource"), str):

727 + raise ValueError("metadata.rate_limit_resource must be a string or null")

728 + if not isinstance(metadata.get("snapshot_path"), str):

729 + raise ValueError("metadata.snapshot_path must be a string")

730 partial_failures = metadata.get("partial_failures")

731 if partial_failures is not None and not isinstance(partial_failures, list):

732 raise ValueError("metadata.partial_failures must be a list when present")

@@ -692,12 +753,17 @@ def main() -> int:

753 client = GitHubClient(github_token)

754 max_results = max(1, min(args.max_results, 1000))

755

695 - date_range = f"{since.date().isoformat()}..{window_end.date().isoformat()}"

696 - new_query = f"created:{date_range} stars:>50"

697 - trending_query = f"pushed:{date_range} stars:>50"

756 + if args.as_of:

757 + created_filter = f"created:{since.date().isoformat()}..{window_end.date().isoformat()}"

758 + pushed_filter = f"pushed:{since.date().isoformat()}..{window_end.date().isoformat()}"

759 + else:

760 + created_filter = f"created:>{since.date().isoformat()}"

761 + pushed_filter = f"pushed:>{since.date().isoformat()}"

762 + new_query = f"{created_filter} stars:>50"

763 + trending_query = f"{pushed_filter} stars:>50"

764

765 new_candidates = client.search_repositories(new_query, max_results=max_results)

700 - previous_stars = load_previous_star_snapshot(SNAPSHOT_ROOT, output_path.parent, week)

766 + previous_stars = load_previous_star_snapshot(SNAPSHOT_ROOT, week, output_path.parent, RAW_ROOT)

767 trending_candidates = client.search_repositories(trending_query, max_results=max_results)

768

769 new_repos, new_filters = collect_repositories(client, new_candidates)

@@ -708,6 +774,8 @@ def main() -> int:

774 trending_cutoff=since,

775 )

776

777 + # Keep star snapshots broader than the filtered payload so future reruns can still compute deltas

778 + # even when a repo is later excluded as low-signal or missing a README.

779 star_snapshot = build_star_snapshot(new_candidates, trending_candidates)

780 snapshot_payload = {

781 "week": week,

@@ -728,7 +796,7 @@ def main() -> int:

796 "stale_cache_hits": client.stale_cache_hits,

797 "rate_limit_limit": client.rate_limit_limit,

798 "rate_limit_remaining": client.rate_limit_remaining,

731 - "rate_limit_reset": rate_limit_reset_text(client.rate_limit_reset),

799 + "rate_limit_reset": client.rate_limit_reset,

800 "rate_limit_resource": client.rate_limit_resource,

801 "partial_failures": client.errors,

802 "filter_summary": {

@@ -749,7 +817,7 @@ def main() -> int:

817 f"Wrote {output_path} with {len(new_repos)} new repos and {len(trending_repos)} trending repos, "

818 f"saved {snapshot_path}, used {client.api_calls_used} API calls, and served {client.cache_hits} cache hits."

819 )

752 - return 0 if (new_candidates or trending_candidates or not client.errors) else 1

820 + return 1 if client.errors else 0

821

822

823 if __name__ == "__main__":

tests/test_crawl.py new

+173

@@ -0,0 +1,173 @@

1 +import tempfile

2 +import unittest

3 +from argparse import Namespace

4 +from pathlib import Path

5 +from unittest import mock

6 +

7 +import scripts.crawl as crawl

8 +

9 +

10 +class CrawlTests(unittest.TestCase):

11 + def test_significance_skip_reason_allows_common_description_terms(self) -> None:

12 + repo = {

13 + "name": "awesome-tool",

14 + "description": "Includes sample data and a live demo for deployment.",

15 + "topics": ["ai", "demo"],

16 + "fork": False,

17 + "is_template": False,

18 + }

19 +

20 + self.assertIsNone(crawl.significance_skip_reason(repo))

21 +

22 + def test_significance_skip_reason_still_flags_name_tokens(self) -> None:

23 + repo = {

24 + "name": "starter-kit",

25 + "description": "Production-ready auth service.",

26 + "topics": ["ai"],

27 + "fork": False,

28 + "is_template": False,

29 + }

30 +

31 + self.assertEqual(crawl.significance_skip_reason(repo), "low_signal_keyword")

32 +

33 + def test_get_json_preserves_payload_contract(self) -> None:

34 + client = crawl.GitHubClient("token")

35 + entry = crawl.CacheEntry(status=200, payload={"ok": True}, headers={}, fetched_at=crawl.utc_now())

36 +

37 + with mock.patch.object(client, "get_json_entry", return_value=entry):

38 + self.assertEqual(client.get_json("https://example.com"), {"ok": True})

39 +

40 + def test_load_previous_star_snapshot_checks_all_raw_dirs_and_logs_failures(self) -> None:

41 + tests_root = Path(__file__).resolve().parent

42 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

43 + base = Path(tmpdir)

44 + snapshot_dir = base / "snapshots"

45 + raw_default_dir = base / "raw-default"

46 + custom_output_dir = base / "custom-output"

47 + snapshot_dir.mkdir()

48 + raw_default_dir.mkdir()

49 + custom_output_dir.mkdir()

50 +

51 + (snapshot_dir / "2026-W21-stars.json").write_text(

52 + '{"week": "2026-W21", "stars": {"owner/current": 1}}\n', encoding="utf-8"

53 + )

54 + (raw_default_dir / "2026-W20.json").write_text("{not-json}\n", encoding="utf-8")

55 + (raw_default_dir / "2026-W19.json").write_text(

56 + '{"week": "2026-W19", "stars": {"owner/older": 42}}\n', encoding="utf-8"

57 + )

58 +

59 + with mock.patch.object(crawl, "log") as log_mock:

60 + stars = crawl.load_previous_star_snapshot(snapshot_dir, "2026-W21", custom_output_dir, raw_default_dir)

61 +

62 + self.assertEqual(stars, {"owner/older": 42})

63 + logged = "\n".join(call.args[0] for call in log_mock.call_args_list)

64 + self.assertIn("invalid JSON", logged)

65 +

66 + def test_validate_payload_accepts_rate_limit_metadata_schema(self) -> None:

67 + payload = {

68 + "week": "2026-W21",

69 + "crawled_at": "2026-05-18T10:00:00Z",

70 + "new_repos": [],

71 + "trending_repos": [],

72 + "signals": {"top_topics": []},

73 + "metadata": {

74 + "api_calls_used": 1,

75 + "cache_hits": 2,

76 + "stale_cache_hits": 3,

77 + "rate_limit_limit": 5000,

78 + "rate_limit_remaining": 4990,

79 + "rate_limit_reset": 1747562400,

80 + "rate_limit_resource": "search",

81 + "partial_failures": [],

82 + "snapshot_path": "data/snapshots/2026-W21-stars.json",

83 + },

84 + }

85 +

86 + crawl.validate_payload(payload)

87 +

88 + def test_pause_for_rate_limit_cools_down_without_reset_hint(self) -> None:

89 + client = crawl.GitHubClient("token")

90 + client.rate_limit_limit = 5000

91 + client.rate_limit_remaining = 0

92 + client.rate_limit_resource = "core"

93 +

94 + with mock.patch("scripts.crawl.time.sleep") as sleep_mock:

95 + client._pause_for_rate_limit("https://example.com")

96 +

97 + sleep_mock.assert_called_once()

98 +

99 + def test_main_uses_open_ended_queries_for_live_runs(self) -> None:

100 + queries: list[str] = []

101 +

102 + class FakeClient:

103 + def __init__(self, token: str) -> None:

104 + self.token = token

105 + self.api_calls_used = 0

106 + self.cache_hits = 0

107 + self.stale_cache_hits = 0

108 + self.rate_limit_limit = None

109 + self.rate_limit_remaining = None

110 + self.rate_limit_reset = None

111 + self.rate_limit_resource = None

112 + self.errors = []

113 +

114 + def search_repositories(self, query: str, *, max_results: int = 1000):

115 + queries.append(query)

116 + return []

117 +

118 + def has_readme(self, full_name: str) -> bool:

119 + return True

120 +

121 + args = Namespace(since="2026-05-11", as_of=None, max_results=25, output="data/raw/test-live.json")

122 + with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(

123 + "os.environ", {"GITHUB_TOKEN": "token"}, clear=False

124 + ), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(

125 + crawl, "load_previous_star_snapshot", return_value={}

126 + ), mock.patch.object(crawl, "write_payload"), mock.patch.object(crawl, "print"):

127 + exit_code = crawl.main()

128 +

129 + self.assertEqual(exit_code, 0)

130 + self.assertEqual(queries, ["created:>2026-05-11 stars:>50", "pushed:>2026-05-11 stars:>50"])

131 +

132 + def test_main_uses_bounded_queries_for_backfills_and_fails_on_partial_errors(self) -> None:

133 + queries: list[str] = []

134 +

135 + class FakeClient:

136 + def __init__(self, token: str) -> None:

137 + self.token = token

138 + self.api_calls_used = 0

139 + self.cache_hits = 0

140 + self.stale_cache_hits = 0

141 + self.rate_limit_limit = None

142 + self.rate_limit_remaining = None

143 + self.rate_limit_reset = None

144 + self.rate_limit_resource = None

145 + self.errors = ["README lookup failed"]

146 +

147 + def search_repositories(self, query: str, *, max_results: int = 1000):

148 + queries.append(query)

149 + return []

150 +

151 + def has_readme(self, full_name: str) -> bool:

152 + return True

153 +

154 + args = Namespace(since="2026-05-11", as_of="2026-05-18", max_results=25, output="data/raw/test-backfill.json")

155 + with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(

156 + "os.environ", {"GITHUB_TOKEN": "token"}, clear=False

157 + ), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(

158 + crawl, "load_previous_star_snapshot", return_value={}

159 + ), mock.patch.object(crawl, "write_payload"), mock.patch.object(crawl, "print"):

160 + exit_code = crawl.main()

161 +

162 + self.assertEqual(exit_code, 1)

163 + self.assertEqual(

164 + queries,

165 + [

166 + "created:2026-05-11..2026-05-18 stars:>50",

167 + "pushed:2026-05-11..2026-05-18 stars:>50",

168 + ],

169 + )

170 +

171 +

172 +if __name__ == "__main__":

173 + unittest.main()