@cryptotaxi247 / SquadScope / commits / b8ccc97

Reuse same-day source artifacts safely on reruns (#282)

* fix: reuse same-day crawl artifacts Add safe same-day reuse for GitHub raw crawl artifacts and per-source external news artifacts with date, window, config, schema, and checksum guards. Preserve recrawl-by-default behavior when provenance is missing or stale, and expose reuse provenance in publish manifests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: preserve structured same-day reuse provenance Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: add github reuse source id Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Juan Manuel Servera committed Jun 6, 2026 at 12:29 UTC b8ccc9732acdf855027e0d249a3e5cd60fc1c318

7 files changed +879 -13

.github/workflows/crawl-and-publish.yml

+20

@@ -90,6 +90,26 @@ jobs:

90 name: crawl-cache

91 path: data/cache/

93 + - name: Restore previous raw data for safe same-day reuse

94 + if: steps.previous-cache-run.outputs.run_id != ''

95 + continue-on-error: true

96 + uses: actions/download-artifact@v4

97 + with:

98 + github-token: ${{ secrets.GITHUB_TOKEN }}

99 + run-id: ${{ steps.previous-cache-run.outputs.run_id }}

100 + name: raw-data

101 + path: data/raw/

102 +

103 + - name: Restore previous snapshots for safe same-day reuse

104 + if: steps.previous-cache-run.outputs.run_id != ''

105 + continue-on-error: true

106 + uses: actions/download-artifact@v4

107 + with:

108 + github-token: ${{ secrets.GITHUB_TOKEN }}

109 + run-id: ${{ steps.previous-cache-run.outputs.run_id }}

110 + name: crawl-snapshots

111 + path: data/snapshots/

112 +

113 - name: Set up Python

114 uses: actions/setup-python@v5

115 with:

scripts/crawl.py

+134

@@ -90,6 +90,7 @@ DEFAULT_HEADERS = {

90 "X-GitHub-Api-Version": "2022-11-28",

91 "User-Agent": "SquadScope-Crawler/1.0",

92 }

93 +GITHUB_SOURCE_ID = "github-search"

96 def log(message: str) -> None:

@@ -490,6 +491,11 @@ def parse_args() -> argparse.Namespace:

491 help="Path to a topic YAML config file (e.g. squadscope.topic.yml). "

492 "When provided, queries are read from the config instead of using hardcoded defaults.",

493 )

494 + parser.add_argument(

495 + "--force-refresh",

496 + action="store_true",

497 + help="Refresh GitHub data even when a same-day raw artifact is reusable.",

498 + )

499 return parser.parse_args()

500

501

@@ -552,6 +558,110 @@ def decode_json_body(body: str) -> Any:

558 return {"message": body.strip()}

559

560

561 +def sha256_text(value: str) -> str:

562 + return hashlib.sha256(value.encode("utf-8")).hexdigest()

563 +

564 +

565 +def sha256_file(path: Path) -> str | None:

566 + if not path.exists() or not path.is_file():

567 + return None

568 + digest = hashlib.sha256()

569 + with path.open("rb") as handle:

570 + for chunk in iter(lambda: handle.read(1024 * 1024), b""):

571 + digest.update(chunk)

572 + return digest.hexdigest()

573 +

574 +

575 +def github_schema_checksum() -> str:

576 + contract = {

577 + "schema": "github_raw_v1",

578 + "top_level": ["week", "crawled_at", "new_repos", "trending_repos", "signals", "metadata"],

579 + "metadata": ["crawl_window", "crawl_config_checksum", "schema_checksum", "artifact_checksum", "same_day_reuse"],

580 + }

581 + return sha256_text(json.dumps(contract, sort_keys=True, separators=(",", ":")))

582 +

583 +

584 +def github_crawl_config_checksum(args: argparse.Namespace, since: datetime, window_end: datetime, max_results: int) -> str:

585 + config_digest = sha256_file(Path(args.config)) if args.config else None

586 + payload = {

587 + "since": since.date().isoformat(),

588 + "until": window_end.date().isoformat(),

589 + "as_of": args.as_of,

590 + "max_results": max_results,

591 + "topic": args.topic,

592 + "config": args.config,

593 + "config_sha256": config_digest,

594 + }

595 + return sha256_text(json.dumps(payload, sort_keys=True, separators=(",", ":")))

596 +

597 +

598 +def github_artifact_checksum(payload: dict[str, Any]) -> str:

599 + candidate = dict(payload)

600 + candidate.pop("crawled_at", None)

601 + metadata = dict(candidate.get("metadata", {}))

602 + metadata.pop("artifact_checksum", None)

603 + metadata.pop("same_day_reuse", None)

604 + candidate["metadata"] = metadata

605 + return sha256_text(json.dumps(candidate, sort_keys=True, separators=(",", ":"), ensure_ascii=False))

606 +

607 +

608 +def load_reusable_github_payload(

609 + path: Path,

610 + *,

611 + week: str,

612 + crawled_at: datetime,

613 + since: datetime,

614 + window_end: datetime,

615 + config_checksum: str,

616 +) -> dict[str, Any] | None:

617 + try:

618 + payload = json.loads(path.read_text(encoding="utf-8"))

619 + except (OSError, json.JSONDecodeError):

620 + return None

621 + if not isinstance(payload, dict):

622 + return None

623 + try:

624 + validate_payload(payload)

625 + except ValueError:

626 + return None

627 + metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {}

628 + parsed = None

629 + raw_crawled_at = payload.get("crawled_at")

630 + if isinstance(raw_crawled_at, str):

631 + try:

632 + parsed = datetime.fromisoformat(raw_crawled_at.replace("Z", "+00:00"))

633 + except ValueError:

634 + parsed = None

635 + window = metadata.get("crawl_window")

636 + if (

637 + payload.get("week") != week

638 + or parsed is None

639 + or parsed.astimezone(UTC).date() != crawled_at.astimezone(UTC).date()

640 + or not isinstance(window, dict)

641 + or window.get("since") != since.date().isoformat()

642 + or window.get("until") != window_end.date().isoformat()

643 + or metadata.get("crawl_config_checksum") != config_checksum

644 + or metadata.get("schema_checksum") != github_schema_checksum()

645 + or metadata.get("artifact_checksum") != github_artifact_checksum(payload)

646 + ):

647 + return None

648 + metadata["same_day_reuse"] = {

649 + "status": "reused",

650 + "source": "github",

651 + "source_id": GITHUB_SOURCE_ID,

652 + "original_run_id": metadata.get("run_id", ""),

653 + "original_crawled_at": payload.get("crawled_at"),

654 + "reused_at": iso_timestamp(crawled_at),

655 + "week": week,

656 + "crawl_window": window,

657 + "crawl_config_checksum": config_checksum,

658 + "schema_checksum": github_schema_checksum(),

659 + "content_checksum": metadata.get("artifact_checksum"),

660 + }

661 + payload["metadata"] = metadata

662 + return payload

663 +

664 +

665 def load_previous_star_snapshot(snapshot_dir: Path, current_week: str, *raw_dirs: Path) -> dict[str, int]:

666 for snapshot in sorted(snapshot_dir.glob("*-stars.json"), reverse=True):

667 stars, reason = load_star_mapping_details(snapshot, current_week)

@@ -803,6 +913,21 @@ def main() -> int:

913 snapshot_path = topic_snapshots / f"{week}-stars.json"

914 client = GitHubClient(github_token, cache_dir=topic_cache)

915 max_results = max(1, min(args.max_results, 1000))

916 + config_checksum = github_crawl_config_checksum(args, since, window_end, max_results)

917 +

918 + if not getattr(args, "force_refresh", False):

919 + reusable = load_reusable_github_payload(

920 + output_path,

921 + week=week,

922 + crawled_at=crawled_at,

923 + since=since,

924 + window_end=window_end,

925 + config_checksum=config_checksum,

926 + )

927 + if reusable is not None:

928 + write_payload(output_path, reusable)

929 + print(f"Reused same-day GitHub raw artifact {output_path}; used 0 API calls.")

930 + return 0

931

932 if args.config:

933 template_vars = {

@@ -866,6 +991,14 @@ def main() -> int:

991 "rate_limit_reset": client.rate_limit_reset,

992 "rate_limit_resource": client.rate_limit_resource,

993 "partial_failures": client.errors,

994 + "run_id": os.environ.get("GITHUB_RUN_ID", "local"),

995 + "crawl_window": {

996 + "since": since.date().isoformat(),

997 + "until": window_end.date().isoformat(),

998 + },

999 + "crawl_config_checksum": config_checksum,

1000 + "schema_checksum": github_schema_checksum(),

1001 + "same_day_reuse": {"status": "not_reused", "source": "github", "source_id": GITHUB_SOURCE_ID},

1002 "filter_summary": {

1003 "new_repos": new_filters,

1004 "trending_repos": trending_filters,

@@ -873,6 +1006,7 @@ def main() -> int:

1006 "snapshot_path": snapshot_path.as_posix(),

1007 },

1008 }

1009 + payload["metadata"]["artifact_checksum"] = github_artifact_checksum(payload)

1010 validate_payload(payload)

1011 write_payload(output_path, payload)

1012 write_payload(snapshot_path, snapshot_payload)

scripts/publish_manifest.py

+10

@@ -78,6 +78,8 @@ def same_day_reuse_status(payload: dict[str, Any] | None) -> dict[str, Any]:

78 if not isinstance(metadata, dict):

79 metadata = {}

80 explicit = metadata.get("same_day_reuse") or metadata.get("same_day_reuse_status")

81 + if isinstance(explicit, dict):

82 + return dict(explicit)

83 if explicit:

84 return {"status": str(explicit), "source": "artifact-metadata"}

85 return {

@@ -130,6 +132,14 @@ def artifact_entry(role: str, path: Path, week: str) -> dict[str, Any]:

132 }

133 if "source_status" in metadata:

134 entry["source_status"] = metadata["source_status"]

135 + if "source_reuse_summary" in metadata:

136 + entry["source_reuse_summary"] = metadata["source_reuse_summary"]

137 + if "source_artifact_provenance" in metadata:

138 + entry["source_artifact_provenance"] = metadata["source_artifact_provenance"]

139 + if "source_config_checksum" in metadata:

140 + entry["source_config_checksum"] = metadata["source_config_checksum"]

141 + if "schema_checksum" in metadata:

142 + entry["schema_checksum"] = metadata["schema_checksum"]

143 if "sources_requested" in metadata:

144 entry["sources_requested"] = metadata["sources_requested"]

145 entry["sources_succeeded"] = metadata.get("sources_succeeded", [])

scripts/techcrunch_crawler.py

+297 -11

@@ -15,6 +15,7 @@ import argparse

15 import hashlib

16 import ipaddress

17 import json

18 +import os

19 import re

20 import sys

21 import time

@@ -166,6 +167,223 @@ def source_config_checksum(sources: list[NewsSourceConfig]) -> str:

167 return hashlib.sha256(payload.encode("utf-8")).hexdigest()

168

169

170 +def schema_checksum() -> str:

171 + """Return a stable checksum for the external-news artifact contract."""

172 + schema_contract = {

173 + "schema_version": CANONICAL_SCHEMA_VERSION,

174 + "top_level": ["schema_version", "week", "source", "crawled_at", "crawl_window", "articles", "metadata"],

175 + "metadata": [

176 + "source_config_checksum",

177 + "schema_checksum",

178 + "sources_requested",

179 + "sources_succeeded",

180 + "sources_failed",

181 + "source_status",

182 + "source_reuse_summary",

183 + "source_artifact_provenance",

184 + "total_articles",

185 + "relevant_articles",

186 + "dedupe_count",

187 + "errors",

188 + "artifact_checksum",

189 + ],

190 + }

191 + payload = json.dumps(schema_contract, sort_keys=True, separators=(",", ":"))

192 + return hashlib.sha256(payload.encode("utf-8")).hexdigest()

193 +

194 +

195 +def source_content_checksum(source_id: str, articles: list[dict[str, Any]]) -> str:

196 + """Return a stable checksum for one source's article payload."""

197 + source_articles = [

198 + article for article in articles

199 + if article.get("source") == source_id or source_id in article.get("sources", [])

200 + ]

201 + payload = json.dumps(source_articles, sort_keys=True, separators=(",", ":"), ensure_ascii=False)

202 + return hashlib.sha256(payload.encode("utf-8")).hexdigest()

203 +

204 +

205 +def parse_iso_datetime(value: Any) -> datetime | None:

206 + if not isinstance(value, str) or not value.strip():

207 + return None

208 + candidate = value.strip()

209 + if candidate.endswith("Z"):

210 + candidate = f"{candidate[:-1]}+00:00"

211 + try:

212 + parsed = datetime.fromisoformat(candidate)

213 + except ValueError:

214 + return None

215 + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)

216 +

217 +

218 +def _load_json_object(path: Path) -> dict[str, Any] | None:

219 + try:

220 + payload = json.loads(path.read_text(encoding="utf-8"))

221 + except (OSError, json.JSONDecodeError):

222 + return None

223 + return payload if isinstance(payload, dict) else None

224 +

225 +

226 +def _same_window(payload: dict[str, Any], since: datetime, until: datetime) -> bool:

227 + window = payload.get("crawl_window")

228 + return isinstance(window, dict) and window.get("since") == iso_timestamp(since) and window.get("until") == iso_timestamp(until)

229 +

230 +

231 +def plan_source_reuse(

232 + previous_path: Path,

233 + sources: list[NewsSourceConfig],

234 + *,

235 + now: datetime,

236 + since: datetime,

237 + until: datetime,

238 + config_checksum: str,

239 + forced_sources: set[str] | None = None,

240 +) -> tuple[list[dict[str, Any]], list[NewsSourceConfig], list[dict[str, Any]], list[dict[str, Any]], list[dict[str, str]]]:

241 + """Load eligible same-day source artifacts and return reused articles plus sources to crawl."""

242 + forced = forced_sources or set()

243 + requested = {source.name for source in sources}

244 + pending: list[NewsSourceConfig] = []

245 + reused_articles: list[dict[str, Any]] = []

246 + summary: list[dict[str, Any]] = []

247 + provenance: list[dict[str, Any]] = []

248 + stale_reasons: list[str] = []

249 + previous = _load_json_object(previous_path) if previous_path.exists() else None

250 + expected_schema_checksum = schema_checksum()

251 +

252 + if previous is None:

253 + stale_reasons = ["missing previous artifact" if not previous_path.exists() else "previous artifact is not valid JSON"]

254 + else:

255 + crawled_at = parse_iso_datetime(previous.get("crawled_at"))

256 + metadata = previous.get("metadata") if isinstance(previous.get("metadata"), dict) else {}

257 + try:

258 + validate_canonical_output(previous)

259 + except ValueError as exc:

260 + stale_reasons.append(str(exc))

261 + if previous.get("week") != week_slug(now):

262 + stale_reasons.append(f"week mismatch: expected {week_slug(now)}, found {previous.get('week')!r}")

263 + if crawled_at is None or crawled_at.astimezone(UTC).date() != now.astimezone(UTC).date():

264 + stale_reasons.append("crawled_at is not from the current UTC day")

265 + if not _same_window(previous, since, until):

266 + stale_reasons.append("crawl_window mismatch")

267 + if metadata.get("source_config_checksum") != config_checksum:

268 + stale_reasons.append("source_config_checksum mismatch")

269 + if metadata.get("schema_checksum") != expected_schema_checksum:

270 + stale_reasons.append("schema_checksum mismatch")

271 +

272 + previous_metadata = previous.get("metadata", {}) if isinstance(previous, dict) and isinstance(previous.get("metadata"), dict) else {}

273 + previous_statuses = {

274 + str(status.get("source")): status

275 + for status in previous_metadata.get("source_status", [])

276 + if isinstance(status, dict) and status.get("source") in requested

277 + }

278 + previous_articles = [

279 + article for article in (previous.get("articles", []) if isinstance(previous, dict) else [])

280 + if isinstance(article, dict)

281 + ]

282 + previous_run_id = str(previous_metadata.get("run_id") or "")

283 + previous_checksum = previous_metadata.get("artifact_checksum")

284 +

285 + for source in sources:

286 + source_id = source.name

287 + status = previous_statuses.get(source_id)

288 + reasons = list(stale_reasons)

289 + action = "missing"

290 + if source_id in forced:

291 + action = "forced"

292 + reasons.append("source explicitly refreshed")

293 + elif stale_reasons:

294 + action = "missing" if stale_reasons == ["missing previous artifact"] else "stale"

295 + elif status is None:

296 + action = "missing"

297 + reasons.append("source missing from previous artifact")

298 + elif not status.get("success"):

299 + action = "failed"

300 + reasons.append("previous source crawl failed")

301 + else:

302 + action = "reused"

303 +

304 + matching_articles = [article for article in previous_articles if article.get("source") == source_id]

305 + summary.append({

306 + "source": source_id,

307 + "action": action,

308 + "reused": action == "reused",

309 + "refreshed": action != "reused",

310 + "reasons": reasons,

311 + })

312 + provenance.append({

313 + "source_id": source_id,

314 + "action": action,

315 + "artifact_path": previous_path.as_posix(),

316 + "original_run_id": previous_run_id,

317 + "original_crawled_at": previous.get("crawled_at") if isinstance(previous, dict) else None,

318 + "evaluated_at": iso_timestamp(now),

319 + "date": now.astimezone(UTC).date().isoformat(),

320 + "week": week_slug(now),

321 + "crawl_window": {"since": iso_timestamp(since), "until": iso_timestamp(until)},

322 + "source_config_checksum": config_checksum,

323 + "schema_checksum": expected_schema_checksum,

324 + "artifact_checksum": previous_checksum,

325 + "content_checksum": source_content_checksum(source_id, matching_articles),

326 + "reasons": reasons,

327 + })

328 + if action == "reused":

329 + reused_articles.extend(matching_articles)

330 + else:

331 + pending.append(source)

332 +

333 + return reused_articles, pending, summary, provenance, []

334 +

335 +

336 +def merge_reuse_results(

337 + initial_summary: list[dict[str, Any]],

338 + initial_provenance: list[dict[str, Any]],

339 + refreshed_statuses: list[dict[str, Any]],

340 + refreshed_articles: list[dict[str, Any]],

341 + *,

342 + now: datetime,

343 + since: datetime,

344 + until: datetime,

345 + config_checksum: str,

346 + output_path: Path,

347 + run_id: str,

348 +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:

349 + """Update reuse plan entries with the result of sources crawled in this run."""

350 + summary_by_source = {entry["source"]: dict(entry) for entry in initial_summary}

351 + provenance_by_source = {entry["source_id"]: dict(entry) for entry in initial_provenance}

352 + for status in refreshed_statuses:

353 + source_id = str(status.get("source"))

354 + if not source_id:

355 + continue

356 + action = "refreshed" if status.get("success") else "failed"

357 + reasons = [] if status.get("success") else [status.get("error_message") or "source crawl failed"]

358 + summary_by_source[source_id] = {

359 + "source": source_id,

360 + "action": action,

361 + "reused": False,

362 + "refreshed": True,

363 + "reasons": reasons,

364 + }

365 + provenance_by_source[source_id] = {

366 + "source_id": source_id,

367 + "action": action,

368 + "artifact_path": output_path.as_posix(),

369 + "original_run_id": run_id,

370 + "original_crawled_at": iso_timestamp(now),

371 + "evaluated_at": iso_timestamp(now),

372 + "date": now.astimezone(UTC).date().isoformat(),

373 + "week": week_slug(now),

374 + "crawl_window": {"since": iso_timestamp(since), "until": iso_timestamp(until)},

375 + "source_config_checksum": config_checksum,

376 + "schema_checksum": schema_checksum(),

377 + "artifact_checksum": None,

378 + "content_checksum": source_content_checksum(source_id, refreshed_articles),

379 + "reasons": reasons,

380 + }

381 + return (

382 + [summary_by_source[source] for source in sorted(summary_by_source)],

383 + [provenance_by_source[source] for source in sorted(provenance_by_source)],

384 + )

385 +

386 +

387 def iso_timestamp(value: datetime) -> str:

388 """Format datetime as ISO 8601 UTC string."""

389 return value.astimezone(UTC).isoformat().replace("+00:00", "Z")

@@ -534,6 +752,9 @@ def _checksum_payload(output: dict[str, Any]) -> dict[str, Any]:

752 """Return the deterministic subset covered by artifact_checksum."""

753 metadata = dict(output.get("metadata", {}))

754 metadata.pop("artifact_checksum", None)

755 + metadata.pop("run_id", None)

756 + metadata.pop("source_reuse_summary", None)

757 + metadata.pop("source_artifact_provenance", None)

758 payload = dict(output)

759 payload["metadata"] = metadata

760 payload.pop("crawled_at", None)

@@ -562,6 +783,9 @@ def build_output(

783 requested_sources: list[str] | None = None,

784 source_statuses: list[dict[str, Any]] | None = None,

785 errors: list[dict[str, str]] | None = None,

786 + source_reuse_summary: list[dict[str, Any]] | None = None,

787 + source_artifact_provenance: list[dict[str, Any]] | None = None,

788 + run_id: str | None = None,

789 ) -> dict[str, Any]:

790 """Build the final output structure with metadata."""

791 articles, dedupe_count = dedupe_articles(articles)

@@ -593,12 +817,16 @@ def build_output(

817 "crawl_window": crawl_window or {},

818 "articles": articles,

819 "metadata": {

820 + "run_id": run_id or "local",

821 "source_count": source_count,

822 "source_config_checksum": source_config_checksum_value or "",

823 + "schema_checksum": schema_checksum(),

824 "sources_requested": sorted(requested),

825 "sources_succeeded": sorted(succeeded or requested),

826 "sources_failed": sorted(failed),

827 "source_status": sorted(statuses, key=lambda status: status["source"]),

828 + "source_reuse_summary": sorted(source_reuse_summary or [], key=lambda item: item["source"]),

829 + "source_artifact_provenance": sorted(source_artifact_provenance or [], key=lambda item: item["source_id"]),

830 "sources_with_articles": dict(sorted(by_source.items())),

831 "total_articles": len(articles),

832 "relevant_articles": len(relevant),

@@ -608,6 +836,9 @@ def build_output(

836 },

837 }

838 output["metadata"]["artifact_checksum"] = artifact_checksum(output)

839 + for entry in output["metadata"]["source_artifact_provenance"]:

840 + if not entry.get("artifact_checksum"):

841 + entry["artifact_checksum"] = output["metadata"]["artifact_checksum"]

842 validate_canonical_output(output)

843 return output

844

@@ -623,10 +854,13 @@ def validate_canonical_output(output: dict[str, Any]) -> None:

854 raise ValueError("External news artifact requires metadata")

855 required = {

856 "source_config_checksum",

857 + "schema_checksum",

858 "sources_requested",

859 "sources_succeeded",

860 "sources_failed",

861 "source_status",

862 + "source_reuse_summary",

863 + "source_artifact_provenance",

864 "total_articles",

865 "relevant_articles",

866 "dedupe_count",

@@ -639,6 +873,8 @@ def validate_canonical_output(output: dict[str, Any]) -> None:

873 expected_checksum = artifact_checksum(output)

874 if metadata.get("artifact_checksum") != expected_checksum:

875 raise ValueError("External news artifact checksum mismatch")

876 + if metadata.get("schema_checksum") != schema_checksum():

877 + raise ValueError("External news artifact schema checksum mismatch")

878

879

880 def main(argv: list[str] | None = None) -> int:

@@ -672,6 +908,17 @@ def main(argv: list[str] | None = None) -> int:

908 default=None,

909 help="Maximum parallel RSS fetches (default: one per source, capped at 8)",

910 )

911 + parser.add_argument(

912 + "--force-refresh",

913 + action="store_true",

914 + help="Refresh all sources even when same-day artifacts are reusable.",

915 + )

916 + parser.add_argument(

917 + "--force-refresh-source",

918 + action="append",

919 + default=[],

920 + help="Refresh one source by id even when its same-day artifact is reusable. Can be repeated.",

921 + )

922 args = parser.parse_args(argv)

923

924 now = datetime.now(UTC)

@@ -687,9 +934,49 @@ def main(argv: list[str] | None = None) -> int:

934 )

935

936 source_configs = load_source_configs(Path(args.sources))

690 - articles, errors, statuses = crawl_sources_parallel(

691 - source_configs, since=since, until=until, max_workers=args.max_workers

937 + if args.output:

938 + out_path = Path(args.output)

939 + else:

940 + out_dir = raw_dir(args.topic)

941 + out_dir.mkdir(parents=True, exist_ok=True)

942 + out_path = out_dir / f"{week_slug(now)}-external-news.json"

943 +

944 + config_checksum = source_config_checksum(source_configs)

945 + force_sources = {source.name for source in source_configs} if args.force_refresh else set(args.force_refresh_source or [])

946 + reused_articles, sources_to_crawl, reuse_summary, provenance, _ = plan_source_reuse(

947 + out_path,

948 + source_configs,

949 + now=now,

950 + since=since,

951 + until=until,

952 + config_checksum=config_checksum,

953 + forced_sources=force_sources,

954 )

955 + refreshed_articles, errors, refreshed_statuses = crawl_sources_parallel(

956 + sources_to_crawl, since=since, until=until, max_workers=args.max_workers

957 + )

958 + articles = [*reused_articles, *refreshed_articles]

959 + reuse_summary, provenance = merge_reuse_results(

960 + reuse_summary,

961 + provenance,

962 + refreshed_statuses,

963 + refreshed_articles,

964 + now=now,

965 + since=since,

966 + until=until,

967 + config_checksum=config_checksum,

968 + output_path=out_path,

969 + run_id=os.environ.get("GITHUB_RUN_ID", "local"),

970 + )

971 + reused_sources = {entry["source_id"] for entry in provenance if entry.get("action") == "reused"}

972 + previous = _load_json_object(out_path) if out_path.exists() else None

973 + previous_metadata = previous.get("metadata", {}) if isinstance(previous, dict) and isinstance(previous.get("metadata"), dict) else {}

974 + previous_statuses = [

975 + {**status, "reused": True}

976 + for status in previous_metadata.get("source_status", [])

977 + if isinstance(status, dict) and status.get("source") in reused_sources

978 + ]

979 + statuses = [*previous_statuses, *refreshed_statuses]

980 output = build_output(

981 articles,

982 crawled_at=now,

@@ -699,27 +986,26 @@ def main(argv: list[str] | None = None) -> int:

986 "since": iso_timestamp(since),

987 "until": iso_timestamp(until),

988 },

702 - source_config_checksum_value=source_config_checksum(source_configs),

989 + source_config_checksum_value=config_checksum,

990 requested_sources=[source.name for source in source_configs],

991 source_statuses=statuses,

992 errors=errors,

993 + source_reuse_summary=reuse_summary,

994 + source_artifact_provenance=provenance,

995 + run_id=os.environ.get("GITHUB_RUN_ID", "local"),

996 )

997

708 - if args.output:

709 - out_path = Path(args.output)

710 - else:

711 - out_dir = raw_dir(args.topic)

712 - out_dir.mkdir(parents=True, exist_ok=True)

713 - out_path = out_dir / f"{week_slug(now)}-external-news.json"

714 -

998 out_path.parent.mkdir(parents=True, exist_ok=True)

999 with open(out_path, "w", encoding="utf-8") as f:

1000 json.dump(output, f, indent=2, ensure_ascii=False)

1001

1002 + reused_count = sum(1 for item in output["metadata"]["source_reuse_summary"] if item["action"] == "reused")

1003 + refreshed_count = sum(1 for item in output["metadata"]["source_reuse_summary"] if item["action"] != "reused")

1004 print(f"Crawled {output['metadata']['total_articles']} articles "

1005 f"from {output['metadata']['source_count']} sources "

1006 f"({output['metadata']['relevant_articles']} relevant, "

722 - f"{output['metadata']['dedupe_count']} deduped) → {out_path}")

1007 + f"{output['metadata']['dedupe_count']} deduped, "

1008 + f"{reused_count} reused, {refreshed_count} refreshed) → {out_path}")

1009 return 0

1010

1011

tests/test_crawl.py

+140

@@ -1,6 +1,8 @@

1 +import json

2 import tempfile

3 import unittest

4 from argparse import Namespace

5 +from datetime import datetime

6 from pathlib import Path

7 from unittest import mock

@@ -168,6 +170,144 @@ class CrawlTests(unittest.TestCase):

170 ],

171 )

172

173 + def test_load_reusable_github_payload_accepts_same_day_matching_artifact(self) -> None:

174 + tests_root = Path(__file__).resolve().parent

175 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

176 + base = Path(tmpdir)

177 + output = base / "data/raw/2026-W21.json"

178 + args = Namespace(since="2026-05-12", as_of="2026-05-19", max_results=25, output=str(output), topic=None, config=None)

179 + since = datetime(2026, 5, 12, tzinfo=crawl.UTC)

180 + window_end = datetime(2026, 5, 19, tzinfo=crawl.UTC)

181 + crawled_at = datetime(2026, 5, 19, 8, 0, tzinfo=crawl.UTC)

182 + checksum = crawl.github_crawl_config_checksum(args, since, window_end, 25)

183 + payload = {

184 + "week": "2026-W21",

185 + "crawled_at": crawl.iso_timestamp(crawled_at),

186 + "new_repos": [],

187 + "trending_repos": [],

188 + "signals": {"top_topics": []},

189 + "metadata": {

190 + "api_calls_used": 1,

191 + "cache_hits": 0,

192 + "stale_cache_hits": 0,

193 + "rate_limit_limit": None,

194 + "rate_limit_remaining": None,

195 + "rate_limit_reset": None,

196 + "rate_limit_resource": None,

197 + "partial_failures": [],

198 + "snapshot_path": "data/snapshots/2026-W21-stars.json",

199 + "crawl_window": {"since": "2026-05-12", "until": "2026-05-19"},

200 + "crawl_config_checksum": checksum,

201 + "schema_checksum": crawl.github_schema_checksum(),

202 + "same_day_reuse": {"status": "not_reused", "source": "github"},

203 + },

204 + }

205 + payload["metadata"]["artifact_checksum"] = crawl.github_artifact_checksum(payload)

206 + crawl.write_payload(output, payload)

207 +

208 + reused = crawl.load_reusable_github_payload(

209 + output,

210 + week="2026-W21",

211 + crawled_at=datetime(2026, 5, 19, 10, 0, tzinfo=crawl.UTC),

212 + since=since,

213 + window_end=window_end,

214 + config_checksum=checksum,

215 + )

216 +

217 + self.assertIsNotNone(reused)

218 + self.assertEqual(reused["metadata"]["same_day_reuse"]["status"], "reused")

219 + self.assertEqual(reused["metadata"]["same_day_reuse"]["source_id"], "github-search")

220 +

221 + def test_main_emits_github_source_id_in_same_day_reuse_metadata(self) -> None:

222 + class FakeClient:

223 + def __init__(self, token: str, **kwargs) -> None:

224 + self.token = token

225 + self.api_calls_used = 0

226 + self.cache_hits = 0

227 + self.stale_cache_hits = 0

228 + self.rate_limit_limit = None

229 + self.rate_limit_remaining = None

230 + self.rate_limit_reset = None

231 + self.rate_limit_resource = None

232 + self.errors = []

233 +

234 + def search_repositories(self, query: str, *, max_results: int = 1000):

235 + return []

236 +

237 + def has_readme(self, full_name: str) -> bool:

238 + return True

239 +

240 + tests_root = Path(__file__).resolve().parent

241 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

242 + output = Path(tmpdir) / "data/raw/2026-W21.json"

243 + args = Namespace(

244 + since="2026-05-12",

245 + as_of="2026-05-19",

246 + max_results=25,

247 + output=str(output),

248 + topic=None,

249 + config=None,

250 + force_refresh=True,

251 + )

252 +

253 + with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(

254 + "os.environ", {"GITHUB_TOKEN": "token"}, clear=False

255 + ), mock.patch.object(crawl, "GitHubClient", FakeClient), mock.patch.object(

256 + crawl, "load_previous_star_snapshot", return_value={}

257 + ), mock.patch.object(

258 + crawl, "snapshots_dir", return_value=Path(tmpdir) / "data/snapshots"

259 + ), mock.patch.object(

260 + crawl, "utc_now", return_value=datetime(2026, 5, 19, 10, 0, tzinfo=crawl.UTC)

261 + ):

262 + exit_code = crawl.main()

263 +

264 + self.assertEqual(exit_code, 0)

265 + payload = json.loads(output.read_text(encoding="utf-8"))

266 + self.assertEqual(payload["metadata"]["same_day_reuse"]["status"], "not_reused")

267 + self.assertEqual(payload["metadata"]["same_day_reuse"]["source"], "github")

268 + self.assertEqual(payload["metadata"]["same_day_reuse"]["source_id"], "github-search")

269 +

270 + def test_load_reusable_github_payload_rejects_config_mismatch(self) -> None:

271 + tests_root = Path(__file__).resolve().parent

272 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

273 + base = Path(tmpdir)

274 + output = base / "data/raw/2026-W21.json"

275 + payload = {

276 + "week": "2026-W21",

277 + "crawled_at": "2026-05-19T08:00:00Z",

278 + "new_repos": [],

279 + "trending_repos": [],

280 + "signals": {"top_topics": []},

281 + "metadata": {

282 + "api_calls_used": 1,

283 + "cache_hits": 0,

284 + "stale_cache_hits": 0,

285 + "rate_limit_limit": None,

286 + "rate_limit_remaining": None,

287 + "rate_limit_reset": None,

288 + "rate_limit_resource": None,

289 + "partial_failures": [],

290 + "snapshot_path": "data/snapshots/2026-W21-stars.json",

291 + "crawl_window": {"since": "2026-05-12", "until": "2026-05-19"},

292 + "crawl_config_checksum": "old",

293 + "schema_checksum": crawl.github_schema_checksum(),

294 + "same_day_reuse": {"status": "not_reused", "source": "github"},

295 + },

296 + }

297 + payload["metadata"]["artifact_checksum"] = crawl.github_artifact_checksum(payload)

298 + crawl.write_payload(output, payload)

299 +

300 + reused = crawl.load_reusable_github_payload(

301 + output,

302 + week="2026-W21",

303 + crawled_at=datetime(2026, 5, 19, 10, 0, tzinfo=crawl.UTC),

304 + since=datetime(2026, 5, 12, tzinfo=crawl.UTC),

305 + window_end=datetime(2026, 5, 19, tzinfo=crawl.UTC),

306 + config_checksum="new",

307 + )

308 +

309 + self.assertIsNone(reused)

310 +

311

312 if __name__ == "__main__":

313 unittest.main()

tests/test_publish_manifest.py

+157 -2

@@ -1,8 +1,11 @@

1 import json

2 import tempfile

3 import unittest

4 +from argparse import Namespace

5 +from datetime import datetime

6 from pathlib import Path

8 +import scripts.crawl as crawl

9 import scripts.publish_manifest as publish_manifest

@@ -11,7 +14,13 @@ CURRENT_DATETIME = "2026-05-18T08:00:00Z"

14 WEEK = "2026-W21"

14 -def write_raw(path: Path, *, week: str = WEEK, crawled_at: str = CURRENT_DATETIME) -> None:

17 +def write_raw(

18 + path: Path,

19 + *,

20 + week: str = WEEK,

21 + crawled_at: str = CURRENT_DATETIME,

22 + metadata: dict | None = None,

23 +) -> None:

24 path.parent.mkdir(parents=True, exist_ok=True)

25 path.write_text(

26 json.dumps(

@@ -20,7 +29,7 @@ def write_raw(path: Path, *, week: str = WEEK, crawled_at: str = CURRENT_DATETIM

29 "crawled_at": crawled_at,

30 "new_repos": [],

31 "trending_repos": [],

23 - "metadata": {"same_day_reuse": "not_reused"},

32 + "metadata": metadata or {"same_day_reuse": "not_reused"},

33 }

34 ),

35 encoding="utf-8",

@@ -162,6 +171,152 @@ class PublishManifestTests(unittest.TestCase):

171 self.assertEqual(payload["source_artifacts"][0]["freshness"]["status"], "stale")

172 self.assertTrue(any("timestamp week mismatch" in reason for reason in payload["promotion"]["reasons"]))

173

174 + def test_structured_same_day_reuse_metadata_remains_machine_readable(self) -> None:

175 + tests_root = Path(__file__).resolve().parent

176 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

177 + base = Path(tmpdir)

178 + raw = base / "data/raw/2026-W21.json"

179 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"

180 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"

181 + reuse_metadata = {

182 + "same_day_reuse": {

183 + "status": "reused",

184 + "source": "github",

185 + "source_id": "github-search",

186 + "original_run_id": "111111",

187 + "original_crawled_at": "2026-05-18T06:00:00Z",

188 + "reused_at": CURRENT_DATETIME,

189 + "week": WEEK,

190 + "crawl_window": {"since": "2026-05-11", "until": "2026-05-18"},

191 + "crawl_config_checksum": "config-sha",

192 + "schema_checksum": "schema-sha",

193 + "content_checksum": "content-sha",

194 + },

195 + "artifact_checksum": "artifact-sha",

196 + }

197 + write_raw(raw, metadata=reuse_metadata)

198 + write_summary(summary)

199 +

200 + publish_manifest.main(

201 + [

202 + "create",

203 + "--week",

204 + WEEK,

205 + "--run-id",

206 + RUN_ID,

207 + "--current-datetime",

208 + CURRENT_DATETIME,

209 + "--summary",

210 + str(summary),

211 + "--published-summary",

212 + str(base / "data/analyzed/2026-W21-summary.md"),

213 + "--raw-json",

214 + str(raw),

215 + "--analysis-source",

216 + "copilot-cli",

217 + "--analysis-model",

218 + "copilot-default",

219 + "--validation-status",

220 + "passed",

221 + "--output",

222 + str(manifest),

223 + ]

224 + )

225 +

226 + reuse = json.loads(manifest.read_text(encoding="utf-8"))["source_artifacts"][0]["same_day_reuse"]

227 + self.assertIsInstance(reuse, dict)

228 + self.assertEqual(reuse["status"], "reused")

229 + self.assertEqual(reuse["source"], "github")

230 + self.assertEqual(reuse["source_id"], "github-search")

231 + self.assertEqual(reuse["original_run_id"], "111111")

232 + self.assertEqual(reuse["original_crawled_at"], "2026-05-18T06:00:00Z")

233 + self.assertEqual(reuse["reused_at"], CURRENT_DATETIME)

234 + self.assertEqual(reuse["crawl_window"]["since"], "2026-05-11")

235 + self.assertEqual(reuse["crawl_config_checksum"], "config-sha")

236 + self.assertEqual(reuse["schema_checksum"], "schema-sha")

237 + self.assertEqual(reuse["content_checksum"], "content-sha")

238 + self.assertNotEqual(reuse["status"], str(dict(reuse)))

239 +

240 + def test_manifest_preserves_source_id_from_crawl_reuse_metadata(self) -> None:

241 + tests_root = Path(__file__).resolve().parent

242 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

243 + base = Path(tmpdir)

244 + raw = base / "data/raw/2026-W21.json"

245 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"

246 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"

247 + since = datetime(2026, 5, 12, tzinfo=crawl.UTC)

248 + window_end = datetime(2026, 5, 19, tzinfo=crawl.UTC)

249 + original_crawled_at = datetime(2026, 5, 19, 8, 0, tzinfo=crawl.UTC)

250 + reused_at = datetime(2026, 5, 19, 10, 0, tzinfo=crawl.UTC)

251 + args = Namespace(since="2026-05-12", as_of="2026-05-19", max_results=25, output=str(raw), topic=None, config=None)

252 + checksum = crawl.github_crawl_config_checksum(args, since, window_end, 25)

253 + payload = {

254 + "week": WEEK,

255 + "crawled_at": crawl.iso_timestamp(original_crawled_at),

256 + "new_repos": [],

257 + "trending_repos": [],

258 + "signals": {"top_topics": []},

259 + "metadata": {

260 + "api_calls_used": 1,

261 + "cache_hits": 0,

262 + "stale_cache_hits": 0,

263 + "rate_limit_limit": None,

264 + "rate_limit_remaining": None,

265 + "rate_limit_reset": None,

266 + "rate_limit_resource": None,

267 + "partial_failures": [],

268 + "run_id": "111111",

269 + "snapshot_path": "data/snapshots/2026-W21-stars.json",

270 + "crawl_window": {"since": "2026-05-12", "until": "2026-05-19"},

271 + "crawl_config_checksum": checksum,

272 + "schema_checksum": crawl.github_schema_checksum(),

273 + "same_day_reuse": {"status": "not_reused", "source": "github", "source_id": crawl.GITHUB_SOURCE_ID},

274 + },

275 + }

276 + payload["metadata"]["artifact_checksum"] = crawl.github_artifact_checksum(payload)

277 + crawl.write_payload(raw, payload)

278 + reused = crawl.load_reusable_github_payload(

279 + raw,

280 + week=WEEK,

281 + crawled_at=reused_at,

282 + since=since,

283 + window_end=window_end,

284 + config_checksum=checksum,

285 + )

286 + self.assertIsNotNone(reused)

287 + crawl.write_payload(raw, reused)

288 + write_summary(summary)

289 +

290 + publish_manifest.main(

291 + [

292 + "create",

293 + "--week",

294 + WEEK,

295 + "--run-id",

296 + RUN_ID,

297 + "--current-datetime",

298 + CURRENT_DATETIME,

299 + "--summary",

300 + str(summary),

301 + "--published-summary",

302 + str(base / "data/analyzed/2026-W21-summary.md"),

303 + "--raw-json",

304 + str(raw),

305 + "--analysis-source",

306 + "copilot-cli",

307 + "--analysis-model",

308 + "copilot-default",

309 + "--validation-status",

310 + "passed",

311 + "--output",

312 + str(manifest),

313 + ]

314 + )

315 +

316 + reuse = json.loads(manifest.read_text(encoding="utf-8"))["source_artifacts"][0]["same_day_reuse"]

317 + self.assertEqual(reuse["status"], "reused")

318 + self.assertEqual(reuse["source_id"], "github-search")

319 +

320

321 if __name__ == "__main__":

322 unittest.main()

tests/test_techcrunch_crawler.py

+121

@@ -550,3 +550,124 @@ class TestExternalNewsSources:

550

551 assert source.last_attempts == DEFAULT_FETCH_RETRIES + 1

552 assert source.last_timeout_seconds == DEFAULT_FETCH_TIMEOUT_SECONDS

553 +

554 +class TestSameDaySourceReuse:

555 + def _sources(self):

556 + return [

557 + NewsSourceConfig("alpha", "https://techcrunch.com/feed/"),

558 + NewsSourceConfig("beta", "https://github.blog/feed/"),

559 + ]

560 +

561 + def _article(self, source, title, url):

562 + return {

563 + "source": source,

564 + "title": title,

565 + "url": url,

566 + "published_at": "2026-05-19T10:00:00Z",

567 + "categories": ["AI"],

568 + "summary": "open source AI framework",

569 + "github_links": [],

570 + "entities": [],

571 + "relevance_score": 0.6,

572 + }

573 +

574 + def _write_previous(self, path, *, crawled_at, statuses=None, articles=None, sources=None):

575 + from scripts.techcrunch_crawler import source_config_checksum

576 + sources = sources or self._sources()

577 + output = build_output(

578 + articles or [self._article("alpha", "Alpha", "https://example.com/alpha")],

579 + crawled_at=crawled_at,

580 + source="external_news",

581 + source_count=len(sources),

582 + crawl_window={

583 + "since": "2026-05-12T00:00:00Z",

584 + "until": "2026-05-19T00:00:00Z",

585 + },

586 + source_config_checksum_value=source_config_checksum(sources),

587 + requested_sources=[source.name for source in sources],

588 + source_statuses=statuses or [

589 + {"source": "alpha", "host": "techcrunch.com", "success": True, "attempts": 1, "timeout_seconds": 15, "total_articles": 1, "relevant_articles": 1, "github_links_found": 0, "started_at": "2026-05-19T08:00:00Z", "ended_at": "2026-05-19T08:00:01Z", "duration_seconds": 1.0, "error_class": "", "error_message": ""},

590 + {"source": "beta", "host": "github.blog", "success": True, "attempts": 1, "timeout_seconds": 15, "total_articles": 0, "relevant_articles": 0, "github_links_found": 0, "started_at": "2026-05-19T08:00:00Z", "ended_at": "2026-05-19T08:00:01Z", "duration_seconds": 1.0, "error_class": "", "error_message": ""},

591 + ],

592 + source_reuse_summary=[],

593 + source_artifact_provenance=[],

594 + run_id="111",

595 + )

596 + path.write_text(json.dumps(output), encoding="utf-8")

597 +

598 + def test_reuses_successful_same_day_sources(self, tmp_path):

599 + from scripts.techcrunch_crawler import plan_source_reuse, source_config_checksum

600 + sources = self._sources()

601 + path = tmp_path / "external.json"

602 + now = datetime(2026, 5, 19, 9, 0, tzinfo=UTC)

603 + self._write_previous(path, crawled_at=now)

604 +

605 + reused, pending, summary, provenance, _ = plan_source_reuse(

606 + path,

607 + sources,

608 + now=now,

609 + since=datetime(2026, 5, 12, tzinfo=UTC),

610 + until=datetime(2026, 5, 19, tzinfo=UTC),

611 + config_checksum=source_config_checksum(sources),

612 + )

613 +

614 + assert [item["action"] for item in summary] == ["reused", "reused"]

615 + assert pending == []

616 + assert [article["title"] for article in reused] == ["Alpha"]

617 + assert provenance[0]["original_run_id"] == "111"

618 + assert provenance[0]["content_checksum"]

619 +

620 + def test_rejects_yesterday_artifact_as_stale(self, tmp_path):

621 + from scripts.techcrunch_crawler import plan_source_reuse, source_config_checksum

622 + sources = self._sources()

623 + path = tmp_path / "external.json"

624 + self._write_previous(path, crawled_at=datetime(2026, 5, 18, 9, 0, tzinfo=UTC))

625 +

626 + reused, pending, summary, _, _ = plan_source_reuse(

627 + path,

628 + sources,

629 + now=datetime(2026, 5, 19, 9, 0, tzinfo=UTC),

630 + since=datetime(2026, 5, 12, tzinfo=UTC),

631 + until=datetime(2026, 5, 19, tzinfo=UTC),

632 + config_checksum=source_config_checksum(sources),

633 + )

634 +

635 + assert reused == []

636 + assert [source.name for source in pending] == ["alpha", "beta"]

637 + assert {item["action"] for item in summary} == {"stale"}

638 +

639 + def test_partial_rerun_reuses_success_and_fetches_failed(self, tmp_path):

640 + from scripts.techcrunch_crawler import plan_source_reuse, source_config_checksum

641 + sources = self._sources()

642 + path = tmp_path / "external.json"

643 + self._write_previous(

644 + path,

645 + crawled_at=datetime(2026, 5, 19, 9, 0, tzinfo=UTC),

646 + statuses=[

647 + {"source": "alpha", "host": "techcrunch.com", "success": True, "attempts": 1, "timeout_seconds": 15, "total_articles": 1, "relevant_articles": 1, "github_links_found": 0, "started_at": "2026-05-19T08:00:00Z", "ended_at": "2026-05-19T08:00:01Z", "duration_seconds": 1.0, "error_class": "", "error_message": ""},

648 + {"source": "beta", "host": "github.blog", "success": False, "attempts": 2, "timeout_seconds": 15, "total_articles": 0, "relevant_articles": 0, "github_links_found": 0, "started_at": "2026-05-19T08:00:00Z", "ended_at": "2026-05-19T08:00:01Z", "duration_seconds": 1.0, "error_class": "TimeoutError", "error_message": "timeout"},

649 + ],

650 + )

651 +

652 + reused, pending, summary, _, _ = plan_source_reuse(

653 + path,

654 + sources,

655 + now=datetime(2026, 5, 19, 10, 0, tzinfo=UTC),

656 + since=datetime(2026, 5, 12, tzinfo=UTC),

657 + until=datetime(2026, 5, 19, tzinfo=UTC),

658 + config_checksum=source_config_checksum(sources),

659 + )

660 +

661 + assert [article["source"] for article in reused] == ["alpha"]

662 + assert [source.name for source in pending] == ["beta"]

663 + assert {item["source"]: item["action"] for item in summary} == {"alpha": "reused", "beta": "failed"}

664 +

665 + def test_deterministic_fan_in_dedupes_reused_and_refreshed_articles(self):

666 + first = self._article("alpha", "Same", "https://example.com/story/")

667 + second = self._article("beta", "Same mirror", "https://example.com/story")

668 + deduped_once, count_once = dedupe_articles([first, second])

669 + deduped_twice, count_twice = dedupe_articles([second, first])

670 +

671 + assert count_once == count_twice == 1

672 + assert deduped_once == deduped_twice

673 + assert deduped_once[0]["sources"] == ["alpha", "beta"]