@cryptotaxi247 / SquadScope-1 / commits / c66b924

Harden crawler for issue #6

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

jmservera committed May 18, 2026 at 12:04 UTC c66b9240555110d23bf228ec803d2980ec156e6e
3 files changed +471 -125
.squad/agents/bender/history.md
+2
@@ -17,3 +17,5 @@
17 - **2026-05-18T10:06:38.734+02:00:** GitHub Actions can run the standalone `copilot` CLI (`@github/copilot`) in programmatic mode with `copilot -p ...`. The safest documented CI auth flow is a fine-grained PAT with the **Copilot Requests** account permission passed as `COPILOT_GITHUB_TOKEN`; `gh auth token` only exposes an existing `gh` token and `gh-copilot` is deprecated in favor of the standalone CLI. GitHub Models (`models: read`) is the clean fallback if direct Copilot CLI automation proves brittle.
18 - **2026-05-18T10:11:20Z:** Team decided Phase 0 PRD decomposition is final; 24 GitHub issues created (4 investigation + 20 implementation). PRD decomposition captures all decisions. MCP tools can crawl beyond GitHub with remote call allowlist. Ready for issue creation.
19 - **2026-05-18T10:27:35.339+02:00:** The crawler now uses `GET /search/repositories` for both `created:>{last_week_date} stars:>50` and `pushed:>{last_week_date} stars:>50`, comparing current stars against the most recent prior `data/raw/*.json` snapshot when available to estimate weekly star gains. It authenticates with `GITHUB_TOKEN`, paginates up to the GitHub Search API's 1,000-result ceiling, caches README checks in-process, applies exponential backoff with jitter for rate limits, and skips repos whose README lookup is blocked by org SAML enforcement.
20 +- **2026-05-18T10:59:10.800+02:00:** Hardening issue #6 required persistent `data/cache/` response caching, dedicated `data/snapshots/YYYY-WNN-stars.json` star maps, stronger low-signal heuristics (fork/template/tutorial/homework filters), and bounded README retries so a real `GITHUB_TOKEN=$(gh auth token) python3 scripts/crawl.py` run can complete while preserving partial data and logging rate-limit state.
21 +- **2026-05-18T10:59:10Z:** Issue #5 complete. Commit fb14275 (209 new repos, 215 trending in data/raw/2026-W21.json). Ready for Issue #6+. User directive: all future work follows branch → PR → Review → Merge workflow (no direct commits to main).
.squad/decisions/inbox/bender-crawler-hardening.md new
+6
@@ -0,0 +1,6 @@
1 +# Bender crawler hardening note
2 +
3 +- **Date:** 2026-05-18T10:59:10.800+02:00
4 +- **Context:** Issue #6 crawler hardening
5 +- **Decision to review:** Treat README lookups as a degradable signal instead of a hard-stop path. The crawler now caches API responses, saves weekly star snapshots under `data/snapshots/`, logs rate-limit state, and caps README retry delays so partial failures are recorded in metadata instead of blocking the full weekly crawl.
6 +- **Why it matters:** Search queries are cheap, but hundreds of README checks can trigger secondary throttling. Bounded retries plus persistent cache keep Phase 1 crawls finishable and give Farnsworth usable JSON even when GitHub responses are partial.
scripts/crawl.py
+463 -125
@@ -4,12 +4,16 @@
4 from __future__ import annotations
5
6 import argparse
7 +import hashlib
8 import json
9 import os
10 import random
11 +import re
12 +import socket
13 import sys
14 import time
15 from collections import Counter
16 +from dataclasses import dataclass
17 from datetime import UTC, datetime, timedelta
18 from pathlib import Path
19 from typing import Any, Iterable
@@ -17,43 +21,69 @@ from urllib import error, parse, request
21
22 API_ROOT = "https://api.github.com"
23 SEARCH_REPOSITORIES = f"{API_ROOT}/search/repositories"
20 -README_KEYWORDS = {
21 - "homework",
24 +CACHE_ROOT = Path("data/cache")
25 +RAW_ROOT = Path("data/raw")
26 +SNAPSHOT_ROOT = Path("data/snapshots")
27 +RETRYABLE_STATUSES = {403, 429, 500, 502, 503, 504}
28 +LOW_SIGNAL_TOPICS = {
29 "assignment",
23 - "tutorial",
24 - "course",
30 + "assignments",
31 "bootcamp",
26 - "workshop",
32 + "course",
33 + "courses",
34 + "example",
35 + "examples",
36 "exercise",
37 + "homework",
38 "lab",
29 - "lesson",
30 - "leetcode",
31 - "kata",
32 - "template",
39 + "learning",
40 + "practice",
41 "starter",
34 - "example",
35 - "cheatsheet",
36 - "guide",
37 - "demo",
38 - "sample",
39 -}
40 -EXCLUDED_TOPICS = {
42 + "starter-template",
43 + "template",
44 + "templates",
45 "tutorial",
46 "tutorials",
43 - "homework",
47 + "workshop",
48 + "workshops",
49 +}
50 +LOW_SIGNAL_TOKENS = {
51 "assignment",
52 "assignments",
53 + "bootcamp",
54 + "cheatsheet",
55 "course",
56 "courses",
48 - "bootcamp",
49 - "workshop",
50 - "workshops",
57 + "demo",
58 "example",
59 "examples",
53 - "template",
54 - "templates",
60 + "exercise",
61 + "exercises",
62 + "homework",
63 + "kata",
64 + "lab",
65 + "labs",
66 + "leetcode",
67 + "lesson",
68 + "lessons",
69 + "practice",
70 + "sample",
71 "starter",
56 - "starter-template",
72 + "template",
73 + "tutorial",
74 + "tutorials",
75 + "walkthrough",
76 + "workshop",
77 +}
78 +LOW_SIGNAL_PHRASES = {
79 + "course project",
80 + "for beginners",
81 + "getting started tutorial",
82 + "my solution",
83 + "starter template",
84 + "step by step",
85 + "step-by-step",
86 + "study notes",
87 }
88 DEFAULT_HEADERS = {
89 "Accept": "application/vnd.github+json, application/vnd.github.mercy-preview+json",
@@ -62,46 +92,157 @@ DEFAULT_HEADERS = {
92 }
93
94
95 +def log(message: str) -> None:
96 + print(f"[crawl {iso_timestamp(utc_now())}] {message}", file=sys.stderr)
97 +
98 +
99 +@dataclass(slots=True)
100 +class CacheEntry:
101 + status: int
102 + payload: Any
103 + headers: dict[str, str]
104 + fetched_at: datetime
105 + stale: bool = False
106 +
107 +
108 +class ResponseCache:
109 + def __init__(self, root: Path) -> None:
110 + self.root = root
111 + self.root.mkdir(parents=True, exist_ok=True)
112 +
113 + def load(self, key: str, ttl_seconds: int) -> CacheEntry | None:
114 + path = self._path_for(key)
115 + if not path.exists():
116 + return None
117 + try:
118 + payload = json.loads(path.read_text(encoding="utf-8"))
119 + fetched_at = datetime.fromisoformat(payload["fetched_at"].replace("Z", "+00:00"))
120 + headers = payload.get("headers") or {}
121 + age = (utc_now() - fetched_at).total_seconds()
122 + return CacheEntry(
123 + status=int(payload["status"]),
124 + payload=payload.get("payload"),
125 + headers={str(name): str(value) for name, value in headers.items()},
126 + fetched_at=fetched_at,
127 + stale=age > ttl_seconds,
128 + )
129 + except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError):
130 + return None
131 +
132 + def store(self, key: str, *, status: int, payload: Any, headers: dict[str, str]) -> None:
133 + path = self._path_for(key)
134 + path.parent.mkdir(parents=True, exist_ok=True)
135 + cache_payload = {
136 + "status": status,
137 + "fetched_at": iso_timestamp(utc_now()),
138 + "headers": headers,
139 + "payload": payload,
140 + }
141 + path.write_text(json.dumps(cache_payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
142 +
143 + def _path_for(self, key: str) -> Path:
144 + parsed = parse.urlparse(key)
145 + label = parsed.path.strip("/").replace("/", "-") or "root"
146 + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()
147 + return self.root / f"{label}-{digest}.json"
148 +
149 +
150 class GitHubClient:
66 - def __init__(self, token: str, *, timeout: int = 30, max_retries: int = 6) -> None:
151 + def __init__(self, token: str, *, cache_dir: Path = CACHE_ROOT, timeout: int = 30, max_retries: int = 6) -> None:
152 self.token = token
153 self.timeout = timeout
154 self.max_retries = max_retries
155 self.api_calls_used = 0
156 + self.cache_hits = 0
157 + self.stale_cache_hits = 0
158 + self.rate_limit_limit: int | None = None
159 self.rate_limit_remaining: int | None = None
160 self.rate_limit_reset: int | None = None
161 + self.rate_limit_resource: str | None = None
162 + self._last_request_at = 0.0
163 self._readme_cache: dict[str, bool] = {}
164 + self._cache = ResponseCache(cache_dir)
165 + self.errors: list[str] = []
166
167 def _headers(self) -> dict[str, str]:
168 headers = dict(DEFAULT_HEADERS)
169 headers["Authorization"] = f"Bearer {self.token}"
170 return headers
171
80 - def get_json(self, url: str, params: dict[str, Any] | None = None) -> Any:
172 + def get_json(
173 + self,
174 + url: str,
175 + params: dict[str, Any] | None = None,
176 + *,
177 + acceptable_statuses: set[int] | None = None,
178 + ttl_seconds: int | None = None,
179 + allow_stale: bool = True,
180 + max_retries: int | None = None,
181 + max_delay_seconds: float = 300.0,
182 + ) -> CacheEntry:
183 query = f"{url}?{parse.urlencode(params)}" if params else url
184 + accepted = acceptable_statuses or set()
185 + retry_limit = self.max_retries if max_retries is None else max_retries
186 + ttl = ttl_seconds if ttl_seconds is not None else self._cache_ttl(url)
187 + cached = self._cache.load(query, ttl)
188 + if cached and not cached.stale and (cached.status == 200 or cached.status in accepted):
189 + self.cache_hits += 1
190 + return cached
191 +
192 + stale_fallback = None
193 + if cached and allow_stale and (cached.status == 200 or cached.status in accepted):
194 + stale_fallback = CacheEntry(
195 + status=cached.status,
196 + payload=cached.payload,
197 + headers=cached.headers,
198 + fetched_at=cached.fetched_at,
199 + stale=True,
200 + )
201 +
202 attempt = 0
203 while True:
204 + self._pause_for_rate_limit(query)
205 + self._respect_min_interval(url)
206 req = request.Request(query, headers=self._headers())
207 try:
208 with request.urlopen(req, timeout=self.timeout) as response:
209 self.api_calls_used += 1
88 - self._update_rate_limit(response.headers)
89 - payload = response.read().decode("utf-8")
90 - return json.loads(payload)
210 + headers = {name: value for name, value in response.headers.items()}
211 + self._update_rate_limit(headers)
212 + payload = json.loads(response.read().decode("utf-8", errors="replace"))
213 + self._last_request_at = time.monotonic()
214 + self._cache.store(query, status=response.status, payload=payload, headers=self._cache_headers(headers))
215 + self._log_rate_limit(query)
216 + return CacheEntry(response.status, payload, headers, utc_now())
217 except error.HTTPError as exc:
218 self.api_calls_used += 1
93 - self._update_rate_limit(exc.headers)
219 + headers = {name: value for name, value in (exc.headers.items() if exc.headers else [])}
220 + self._update_rate_limit(headers)
221 body = exc.read().decode("utf-8", errors="replace")
95 - if attempt >= self.max_retries or not self._should_retry(exc.code, body):
222 + self._last_request_at = time.monotonic()
223 + payload = decode_json_body(body)
224 + if exc.code in accepted:
225 + self._cache.store(query, status=exc.code, payload=payload, headers=self._cache_headers(headers))
226 + self._log_rate_limit(query)
227 + return CacheEntry(exc.code, payload, headers, utc_now())
228 + if attempt >= retry_limit or not self._should_retry(exc.code, body):
229 + if stale_fallback is not None:
230 + self.stale_cache_hits += 1
231 + log(f"Using stale cache for {query} after HTTP {exc.code}.")
232 + return stale_fallback
233 raise RuntimeError(
234 f"GitHub API request failed with status {exc.code}: {body.strip() or exc.reason}"
235 ) from exc
99 - self._sleep_before_retry(attempt, exc.headers, body)
236 + self._sleep_before_retry(attempt, headers, body, query, retry_limit, max_delay_seconds)
237 attempt += 1
101 - except error.URLError as exc:
102 - if attempt >= self.max_retries:
238 + except (error.URLError, TimeoutError, socket.timeout, json.JSONDecodeError) as exc:
239 + if attempt >= retry_limit:
240 + if stale_fallback is not None:
241 + self.stale_cache_hits += 1
242 + log(f"Using stale cache for {query} after network error: {exc}")
243 + return stale_fallback
244 raise RuntimeError(f"GitHub API request failed: {exc}") from exc
104 - self._sleep_before_retry(attempt, None, str(exc))
245 + self._sleep_before_retry(attempt, None, str(exc), query, retry_limit, max_delay_seconds)
246 attempt += 1
247
248 def search_repositories(self, query: str, *, max_results: int = 1000) -> list[dict[str, Any]]:
@@ -109,21 +250,31 @@ class GitHubClient:
250 per_page = 100
251 max_pages = min((max_results + per_page - 1) // per_page, 10)
252 for page in range(1, max_pages + 1):
112 - payload = self.get_json(
113 - SEARCH_REPOSITORIES,
114 - params={
115 - "q": query,
116 - "sort": "stars",
117 - "order": "desc",
118 - "per_page": per_page,
119 - "page": page,
120 - },
121 - )
122 - items = payload.get("items", [])
123 - if not items:
253 + try:
254 + response = self.get_json(
255 + SEARCH_REPOSITORIES,
256 + params={
257 + "q": query,
258 + "sort": "stars",
259 + "order": "desc",
260 + "per_page": per_page,
261 + "page": page,
262 + },
263 + ttl_seconds=6 * 60 * 60,
264 + )
265 + except RuntimeError as exc:
266 + self.record_error(f"Search failed for '{query}' page {page}: {exc}")
267 break
125 - results.extend(items)
126 - if len(items) < per_page or len(results) >= min(payload.get("total_count", 0), max_results, 1000):
268 + payload = response.payload if isinstance(response.payload, dict) else {}
269 + items = payload.get("items")
270 + if not isinstance(items, list):
271 + self.record_error(f"Malformed search payload for '{query}' page {page}: missing items list")
272 + break
273 + if payload.get("incomplete_results"):
274 + self.record_error(f"GitHub marked search results incomplete for '{query}' page {page}")
275 + results.extend(item for item in items if isinstance(item, dict))
276 + total_count = payload.get("total_count")
277 + if len(items) < per_page or len(results) >= min(int(total_count or 0), max_results, 1000):
278 break
279 return results[:max_results]
280
@@ -132,41 +283,114 @@ class GitHubClient:
283 return self._readme_cache[full_name]
284 url = f"{API_ROOT}/repos/{full_name}/readme"
285 try:
135 - self.get_json(url)
136 - self._readme_cache[full_name] = True
286 + response = self.get_json(
287 + url,
288 + acceptable_statuses={404},
289 + ttl_seconds=24 * 60 * 60,
290 + max_retries=2,
291 + max_delay_seconds=60.0,
292 + )
293 except RuntimeError as exc:
294 message = str(exc)
139 - if "status 404" in message or "SAML enforcement" in message:
295 + if "SAML enforcement" in message:
296 self._readme_cache[full_name] = False
141 - else:
142 - raise
143 - return self._readme_cache[full_name]
297 + return False
298 + raise
299 + has_readme = response.status != 404
300 + self._readme_cache[full_name] = has_readme
301 + return has_readme
302 +
303 + def record_error(self, message: str) -> None:
304 + self.errors.append(message)
305 + log(message)
306 +
307 + def _cache_ttl(self, url: str) -> int:
308 + if "/search/" in url:
309 + return 6 * 60 * 60
310 + if url.endswith("/readme"):
311 + return 24 * 60 * 60
312 + return 12 * 60 * 60
313
314 def _should_retry(self, status: int, body: str) -> bool:
315 lowered = body.lower()
147 - return status in {403, 429, 500, 502, 503, 504} or "secondary rate limit" in lowered
148 -
149 - def _sleep_before_retry(self, attempt: int, headers: Any, body: str) -> None:
150 - reset_at = None
151 - if headers is not None:
152 - remaining = headers.get("X-RateLimit-Remaining")
153 - reset = headers.get("X-RateLimit-Reset")
154 - if remaining == "0" and reset:
155 - try:
156 - reset_at = max(int(reset) - int(time.time()), 1)
157 - except ValueError:
158 - reset_at = None
159 - if reset_at is not None:
160 - delay = reset_at + random.uniform(0.0, 1.0)
161 - else:
162 - delay = min(2**attempt, 60) + random.uniform(0.0, 1.0)
163 - if "secondary rate limit" in body.lower():
164 - delay = max(delay, 5.0 + random.uniform(0.0, 3.0))
316 + return status in RETRYABLE_STATUSES or "secondary rate limit" in lowered
317 +
318 + def _sleep_before_retry(
319 + self,
320 + attempt: int,
321 + headers: dict[str, str] | None,
322 + body: str,
323 + query: str,
324 + retry_limit: int,
325 + max_delay_seconds: float,
326 + ) -> None:
327 + reset_delay = self._reset_delay(headers)
328 + retry_after = None
329 + if headers and headers.get("Retry-After"):
330 + try:
331 + retry_after = max(float(headers["Retry-After"]), 1.0)
332 + except ValueError:
333 + retry_after = None
334 + base_delay = min(2**attempt, 60)
335 + jitter = random.uniform(0.3, 1.7)
336 + delay = retry_after or reset_delay or (base_delay + jitter)
337 + if "secondary rate limit" in body.lower():
338 + delay = max(delay, 8.0 + random.uniform(0.0, 5.0))
339 + delay = min(delay, max_delay_seconds)
340 + log(f"Retrying {query} in {delay:.1f}s (attempt {attempt + 1}/{retry_limit}).")
341 + time.sleep(delay)
342 +
343 + def _respect_min_interval(self, url: str) -> None:
344 + minimum_interval = 0.35 if url.endswith("/readme") else 0.0
345 + if minimum_interval <= 0:
346 + return
347 + elapsed = time.monotonic() - self._last_request_at
348 + if elapsed < minimum_interval:
349 + time.sleep(minimum_interval - elapsed)
350 +
351 + def _pause_for_rate_limit(self, query: str) -> None:
352 + if self.rate_limit_remaining is None or self.rate_limit_limit is None:
353 + return
354 + low_threshold = max(5, min(25, int(self.rate_limit_limit * 0.1)))
355 + critical_threshold = max(3, min(10, int(self.rate_limit_limit * 0.03)))
356 + if self.rate_limit_remaining > low_threshold:
357 + return
358 + reset_delay = self._reset_delay({"X-RateLimit-Reset": str(self.rate_limit_reset)} if self.rate_limit_reset else None)
359 + if reset_delay is None:
360 + return
361 + if self.rate_limit_remaining <= critical_threshold:
362 + delay = min(reset_delay + random.uniform(0.3, 1.5), 300.0)
363 + log(f"Rate limit nearly exhausted before {query}; pausing {delay:.1f}s until reset window.")
364 + time.sleep(delay)
365 + return
366 + delay = min(max(reset_delay / 10, 1.0), 30.0)
367 + log(
368 + f"Rate limit low ({self.rate_limit_remaining}/{self.rate_limit_limit} {self.rate_limit_resource or 'requests'}); "
369 + f"cooling down {delay:.1f}s before {query}."
370 + )
371 time.sleep(delay)
372
167 - def _update_rate_limit(self, headers: Any) -> None:
373 + def _reset_delay(self, headers: dict[str, str] | None) -> float | None:
374 + if not headers:
375 + return None
376 + reset = headers.get("X-RateLimit-Reset")
377 + if not reset:
378 + return None
379 + try:
380 + return max(int(reset) - int(time.time()), 1)
381 + except ValueError:
382 + return None
383 +
384 + def _update_rate_limit(self, headers: dict[str, str] | None) -> None:
385 + limit = headers.get("X-RateLimit-Limit") if headers else None
386 remaining = headers.get("X-RateLimit-Remaining") if headers else None
387 reset = headers.get("X-RateLimit-Reset") if headers else None
388 + resource = headers.get("X-RateLimit-Resource") if headers else None
389 + if limit is not None:
390 + try:
391 + self.rate_limit_limit = int(limit)
392 + except ValueError:
393 + self.rate_limit_limit = None
394 if remaining is not None:
395 try:
396 self.rate_limit_remaining = int(remaining)
@@ -177,6 +401,23 @@ class GitHubClient:
401 self.rate_limit_reset = int(reset)
402 except ValueError:
403 self.rate_limit_reset = None
404 + if resource is not None:
405 + self.rate_limit_resource = resource
406 +
407 + def _log_rate_limit(self, query: str) -> None:
408 + if self.rate_limit_remaining is None:
409 + return
410 + reset_text = rate_limit_reset_text(self.rate_limit_reset)
411 + limit_text = self.rate_limit_limit if self.rate_limit_limit is not None else "?"
412 + resource_text = self.rate_limit_resource or "unknown"
413 + log(
414 + f"Rate limit after {query}: remaining={self.rate_limit_remaining}/{limit_text} "
415 + f"({resource_text}), resets={reset_text}."
416 + )
417 +
418 + def _cache_headers(self, headers: dict[str, str]) -> dict[str, str]:
419 + names = {"Date", "Retry-After", "X-RateLimit-Limit", "X-RateLimit-Remaining", "X-RateLimit-Reset"}
420 + return {name: value for name, value in headers.items() if name in names}
421
422
423 def parse_args() -> argparse.Namespace:
@@ -207,7 +448,7 @@ def utc_now() -> datetime:
448
449
450 def iso_timestamp(value: datetime) -> str:
210 - return value.isoformat().replace("+00:00", "Z")
451 + return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
452
453
454 def week_slug(value: datetime) -> str:
@@ -215,40 +456,81 @@ def week_slug(value: datetime) -> str:
456 return f"{year}-W{week:02d}"
457
458
218 -def previous_snapshot(data_dir: Path, current_week: str) -> dict[str, int]:
219 - snapshots = sorted(data_dir.glob("*.json"), reverse=True)
220 - for snapshot in snapshots:
221 - try:
222 - payload = json.loads(snapshot.read_text(encoding="utf-8"))
223 - except (OSError, json.JSONDecodeError):
224 - continue
225 - if payload.get("week") == current_week:
226 - continue
227 - mapping: dict[str, int] = {}
228 - for section in ("new_repos", "trending_repos"):
229 - for repo in payload.get(section, []):
230 - full_name = repo.get("full_name") or ""
231 - stars = repo.get("stars")
232 - if full_name and isinstance(stars, int):
233 - mapping[full_name] = stars
459 +def rate_limit_reset_text(reset_timestamp: int | None) -> str | None:
460 + if reset_timestamp is None:
461 + return None
462 + return iso_timestamp(datetime.fromtimestamp(reset_timestamp, tz=UTC))
463 +
464 +
465 +def decode_json_body(body: str) -> Any:
466 + try:
467 + return json.loads(body)
468 + except json.JSONDecodeError:
469 + return {"message": body.strip()}
470 +
471 +
472 +def load_previous_star_snapshot(snapshot_dir: Path, raw_dir: Path, current_week: str) -> dict[str, int]:
473 + for snapshot in sorted(snapshot_dir.glob("*-stars.json"), reverse=True):
474 + stars = load_star_mapping(snapshot, current_week)
475 + if stars:
476 + return stars
477 + for snapshot in sorted(raw_dir.glob("*.json"), reverse=True):
478 + stars = load_star_mapping(snapshot, current_week)
479 + if stars:
480 + return stars
481 + return {}
482 +
483 +
484 +def load_star_mapping(path: Path, current_week: str) -> dict[str, int]:
485 + try:
486 + payload = json.loads(path.read_text(encoding="utf-8"))
487 + except (OSError, json.JSONDecodeError):
488 + return {}
489 + if payload.get("week") == current_week:
490 + return {}
491 + stars = payload.get("stars")
492 + if isinstance(stars, dict):
493 + mapping = {name: int(value) for name, value in stars.items() if isinstance(value, int)}
494 if mapping:
495 return mapping
236 - return {}
496 + mapping: dict[str, int] = {}
497 + for section in ("new_repos", "trending_repos"):
498 + for repo in payload.get(section, []):
499 + full_name = repo.get("full_name") or ""
500 + stars_value = repo.get("stars")
501 + if full_name and isinstance(stars_value, int):
502 + mapping[full_name] = stars_value
503 + return mapping
504
505
239 -def looks_insignificant(repo: dict[str, Any]) -> bool:
506 +def tokenize(text: str) -> set[str]:
507 + return set(re.findall(r"[a-z0-9]+", text.lower()))
508 +
509 +
510 +def significance_skip_reason(repo: dict[str, Any]) -> str | None:
511 if repo.get("fork"):
241 - return True
512 + return "fork"
513 + if repo.get("is_template"):
514 + return "template_repo"
515 description = (repo.get("description") or "").strip()
516 if not description:
244 - return True
245 - lowered_description = description.lower()
246 - lowered_name = str(repo.get("name") or "").lower()
247 - topics = {topic.lower() for topic in repo.get("topics") or []}
248 - combined = " ".join([lowered_name, lowered_description, " ".join(sorted(topics))])
249 - if topics & EXCLUDED_TOPICS:
250 - return True
251 - return any(keyword in combined for keyword in README_KEYWORDS)
517 + return "missing_description"
518 + topics = {str(topic).lower() for topic in repo.get("topics") or []}
519 + if topics & LOW_SIGNAL_TOPICS:
520 + return "low_signal_topic"
521 + combined_text = " ".join(
522 + [
523 + str(repo.get("name") or ""),
524 + description,
525 + " ".join(sorted(topics)),
526 + ]
527 + ).lower()
528 + combined_tokens = tokenize(combined_text)
529 + if combined_tokens & LOW_SIGNAL_TOKENS:
530 + return "low_signal_keyword"
531 + if any(phrase in combined_text for phrase in LOW_SIGNAL_PHRASES):
532 + return "low_signal_phrase"
533 + return None
534
535
536 def to_repo_record(repo: dict[str, Any], *, stars_gained: int | None = None) -> dict[str, Any]:
@@ -262,7 +544,7 @@ def to_repo_record(repo: dict[str, Any], *, stars_gained: int | None = None) ->
544 "stars": repo.get("stargazers_count"),
545 "forks": repo.get("forks_count"),
546 "created_at": repo.get("created_at"),
265 - "topics": sorted(repo.get("topics") or []),
547 + "topics": sorted(str(topic).lower() for topic in (repo.get("topics") or [])),
548 "license": license_info.get("spdx_id") or license_info.get("name"),
549 "url": repo.get("html_url"),
550 }
@@ -277,17 +559,32 @@ def collect_repositories(
559 *,
560 previous_stars: dict[str, int] | None = None,
561 trending_cutoff: datetime | None = None,
280 -) -> list[dict[str, Any]]:
562 +) -> tuple[list[dict[str, Any]], dict[str, int]]:
563 collected: list[dict[str, Any]] = []
564 seen: set[str] = set()
565 + filter_stats: Counter[str] = Counter()
566 has_prior_snapshot = bool(previous_stars)
567 for repo in repositories:
568 full_name = repo.get("full_name")
286 - if not full_name or full_name in seen or looks_insignificant(repo):
569 + if not full_name:
570 + filter_stats["missing_full_name"] += 1
571 continue
288 - if not client.has_readme(full_name):
572 + if full_name in seen:
573 + filter_stats["duplicate"] += 1
574 continue
575 seen.add(full_name)
576 + skip_reason = significance_skip_reason(repo)
577 + if skip_reason:
578 + filter_stats[skip_reason] += 1
579 + continue
580 + try:
581 + if not client.has_readme(full_name):
582 + filter_stats["missing_readme"] += 1
583 + continue
584 + except RuntimeError as exc:
585 + filter_stats["readme_lookup_failed"] += 1
586 + client.record_error(f"README lookup failed for {full_name}: {exc}")
587 + continue
588 stars_gained: int | None = None
589 if previous_stars is not None:
590 previous_value = previous_stars.get(full_name)
@@ -305,21 +602,34 @@ def collect_repositories(
602 collected.sort(key=lambda item: (item.get("stars_gained", -1), item["stars"]), reverse=True)
603 else:
604 collected.sort(key=lambda item: item["stars"], reverse=True)
308 - return collected
605 + return collected, dict(filter_stats)
606
607
608 def build_signals(*repo_groups: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
609 topic_counter: Counter[str] = Counter()
610 + merged: dict[str, dict[str, Any]] = {}
611 for group in repo_groups:
612 for repo in group:
315 - topic_counter.update(topic.lower() for topic in repo.get("topics") or [])
316 - top_topics = [
317 - {"topic": topic, "count": count}
318 - for topic, count in topic_counter.most_common(15)
319 - ]
613 + full_name = repo.get("full_name")
614 + if full_name:
615 + merged[full_name] = repo
616 + for repo in merged.values():
617 + topic_counter.update(topic.lower() for topic in repo.get("topics") or [])
618 + top_topics = [{"topic": topic, "count": count} for topic, count in topic_counter.most_common(15)]
619 return {"top_topics": top_topics}
620
621
622 +def build_star_snapshot(*repo_groups: Iterable[dict[str, Any]]) -> dict[str, int]:
623 + stars: dict[str, int] = {}
624 + for group in repo_groups:
625 + for repo in group:
626 + full_name = repo.get("full_name")
627 + value = repo.get("stargazers_count")
628 + if full_name and isinstance(value, int):
629 + stars[full_name] = value
630 + return dict(sorted(stars.items()))
631 +
632 +
633 def validate_payload(payload: dict[str, Any]) -> None:
634 required_top_level = {"week", "crawled_at", "new_repos", "trending_repos", "signals", "metadata"}
635 missing = required_top_level - payload.keys()
@@ -350,8 +660,15 @@ def validate_payload(payload: dict[str, Any]) -> None:
660 metadata = payload["metadata"]
661 if not isinstance(metadata.get("api_calls_used"), int):
662 raise ValueError("metadata.api_calls_used must be an integer")
663 + if not isinstance(metadata.get("cache_hits"), int):
664 + raise ValueError("metadata.cache_hits must be an integer")
665 if metadata.get("rate_limit_remaining") is not None and not isinstance(metadata.get("rate_limit_remaining"), int):
666 raise ValueError("metadata.rate_limit_remaining must be an integer or null")
667 + if metadata.get("rate_limit_limit") is not None and not isinstance(metadata.get("rate_limit_limit"), int):
668 + raise ValueError("metadata.rate_limit_limit must be an integer or null")
669 + partial_failures = metadata.get("partial_failures")
670 + if partial_failures is not None and not isinstance(partial_failures, list):
671 + raise ValueError("metadata.partial_failures must be a list when present")
672
673
674 def write_payload(path: Path, payload: dict[str, Any]) -> None:
@@ -367,32 +684,38 @@ def main() -> int:
684 return 1
685
686 crawled_at = utc_now()
370 - window_end = (
371 - datetime.strptime(args.as_of, "%Y-%m-%d").replace(tzinfo=UTC)
372 - if args.as_of
373 - else crawled_at
374 - )
687 + window_end = datetime.strptime(args.as_of, "%Y-%m-%d").replace(tzinfo=UTC) if args.as_of else crawled_at
688 since = datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=UTC) if args.since else window_end - timedelta(days=7)
689 week = week_slug(window_end)
377 - output_path = Path(args.output) if args.output else Path("data/raw") / f"{week}.json"
690 + output_path = Path(args.output) if args.output else RAW_ROOT / f"{week}.json"
691 + snapshot_path = SNAPSHOT_ROOT / f"{week}-stars.json"
692 client = GitHubClient(github_token)
693 max_results = max(1, min(args.max_results, 1000))
694
381 - new_query = f"created:>{since.date().isoformat()} stars:>50"
382 - trending_query = f"pushed:>{since.date().isoformat()} stars:>50"
695 + date_range = f"{since.date().isoformat()}..{window_end.date().isoformat()}"
696 + new_query = f"created:{date_range} stars:>50"
697 + trending_query = f"pushed:{date_range} stars:>50"
698
699 new_candidates = client.search_repositories(new_query, max_results=max_results)
385 - previous_stars = previous_snapshot(output_path.parent, week)
700 + previous_stars = load_previous_star_snapshot(SNAPSHOT_ROOT, output_path.parent, week)
701 trending_candidates = client.search_repositories(trending_query, max_results=max_results)
702
388 - new_repos = collect_repositories(client, new_candidates)
389 - trending_repos = collect_repositories(
703 + new_repos, new_filters = collect_repositories(client, new_candidates)
704 + trending_repos, trending_filters = collect_repositories(
705 client,
706 trending_candidates,
707 previous_stars=previous_stars,
708 trending_cutoff=since,
709 )
710
711 + star_snapshot = build_star_snapshot(new_candidates, trending_candidates)
712 + snapshot_payload = {
713 + "week": week,
714 + "captured_at": iso_timestamp(crawled_at),
715 + "repository_count": len(star_snapshot),
716 + "stars": star_snapshot,
717 + }
718 +
719 payload = {
720 "week": week,
721 "crawled_at": iso_timestamp(crawled_at),
@@ -401,17 +724,32 @@ def main() -> int:
724 "signals": build_signals(new_repos, trending_repos),
725 "metadata": {
726 "api_calls_used": client.api_calls_used,
727 + "cache_hits": client.cache_hits,
728 + "stale_cache_hits": client.stale_cache_hits,
729 + "rate_limit_limit": client.rate_limit_limit,
730 "rate_limit_remaining": client.rate_limit_remaining,
731 + "rate_limit_reset": rate_limit_reset_text(client.rate_limit_reset),
732 + "rate_limit_resource": client.rate_limit_resource,
733 + "partial_failures": client.errors,
734 + "filter_summary": {
735 + "new_repos": new_filters,
736 + "trending_repos": trending_filters,
737 + },
738 + "snapshot_path": snapshot_path.as_posix(),
739 },
740 }
741 validate_payload(payload)
742 write_payload(output_path, payload)
743 + write_payload(snapshot_path, snapshot_payload)
744 +
745 + if client.errors:
746 + log(f"Completed with {len(client.errors)} partial failure(s).")
747
748 print(
411 - f"Wrote {output_path} with {len(new_repos)} new repos and {len(trending_repos)} trending repos "
412 - f"using {client.api_calls_used} API calls."
749 + f"Wrote {output_path} with {len(new_repos)} new repos and {len(trending_repos)} trending repos, "
750 + f"saved {snapshot_path}, used {client.api_calls_used} API calls, and served {client.cache_hits} cache hits."
751 )
414 - return 0
752 + return 0 if (new_candidates or trending_candidates or not client.errors) else 1
753
754
755 if __name__ == "__main__":