@cryptotaxi247 / SquadScope-1 / commits / 10c2902

Add explicit safe rerun modes and restore controls (#279)

* fix: make weekly reruns explicit and safe Add guarded rerun modes and source refresh policy validation for the weekly crawl workflow. Reuse eligible same-day GitHub and external-news source artifacts while preserving gate-backed promotion behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: address safe rerun review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Harden same-day artifact reuse Treat missing crawler fingerprints as stale when a current code SHA is supplied, and constrain restored snapshot paths to the expected snapshot tree. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Juan Manuel Servera committed Jun 6, 2026 at 22:00 UTC 10c290246db5e788e38ad1e2f22bfc0e1b02fc03
11 files changed +916 -36
.github/workflows/crawl-and-publish.yml
+83 -15
@@ -5,13 +5,33 @@ on:
5 - cron: '30 8 * * 1'
6 workflow_dispatch:
7 inputs:
8 + run_mode:
9 + description: 'Rerun mode: normal is guarded/non-destructive; dry-run/candidate-only never publish; restore requires rebuild_week; force-replace is explicit replacement.'
10 + required: false
11 + default: normal
12 + type: choice
13 + options:
14 + - normal
15 + - dry-run
16 + - restore
17 + - force-replace
18 + - candidate-only
19 + source_refresh_policy:
20 + description: 'Source refresh policy for reruns. Default reuses eligible same-day source artifacts and refreshes missing/stale sources.'
21 + required: false
22 + default: reuse-same-day
23 + type: choice
24 + options:
25 + - reuse-same-day
26 + - refresh-missing-stale
27 + - force-refresh
28 publish_release:
9 - description: 'Create or update the weekly GitHub Release during a manual run.'
29 + description: 'Create or update the weekly GitHub Release during a manual run. Not allowed for dry-run/candidate-only.'
30 required: false
31 default: false
32 type: boolean
33 rebuild_week:
14 - description: 'Rebuild a specific past week (e.g. 2026-W21). Skips crawl, hydrates raw/analyzed from publish, regenerates content+rollups only.'
34 + description: 'Restore a specific past week (e.g. 2026-W21). Requires run_mode=restore; skips crawl and hydrates from publish.'
35 required: false
36 default: ''
37 type: string
@@ -41,6 +61,23 @@ jobs:
61 fetch-depth: 0
62 ref: ${{ github.event.repository.default_branch }}
63
64 + - name: Validate rerun mode
65 + id: rerun-mode
66 + env:
67 + RUN_MODE: ${{ inputs.run_mode || 'normal' }}
68 + SOURCE_REFRESH_POLICY: ${{ inputs.source_refresh_policy || 'reuse-same-day' }}
69 + REBUILD_WEEK: ${{ inputs.rebuild_week || '' }}
70 + PUBLISH_RELEASE: ${{ inputs.publish_release || false }}
71 + run: |
72 + set -euo pipefail
73 + ARGS=(--run-mode "$RUN_MODE" --source-refresh-policy "$SOURCE_REFRESH_POLICY" --rebuild-week "$REBUILD_WEEK" --summary-json data/diagnostics/rerun-mode.json)
74 + if [ "$PUBLISH_RELEASE" = "true" ]; then
75 + ARGS+=(--publish-release)
76 + fi
77 + python3 scripts/rerun_modes.py "${ARGS[@]}"
78 + echo "run_mode=$RUN_MODE" >> "$GITHUB_OUTPUT"
79 + echo "source_refresh_policy=$SOURCE_REFRESH_POLICY" >> "$GITHUB_OUTPUT"
80 +
81 - name: Find latest successful crawl cache
82 id: previous-cache-run
83 uses: actions/github-script@v7
@@ -90,25 +127,25 @@ jobs:
127 name: crawl-cache
128 path: data/cache/
129
93 - - name: Restore previous raw data for safe same-day reuse
94 - if: steps.previous-cache-run.outputs.run_id != ''
130 + - name: Download previous raw artifacts for same-day reuse
131 + if: ${{ steps.previous-cache-run.outputs.run_id != '' && !inputs.rebuild_week }}
132 continue-on-error: true
133 uses: actions/download-artifact@v4
134 with:
135 github-token: ${{ secrets.GITHUB_TOKEN }}
136 run-id: ${{ steps.previous-cache-run.outputs.run_id }}
137 name: raw-data
101 - path: data/raw/
138 + path: .artifact-reuse/raw-data/
139
103 - - name: Restore previous snapshots for safe same-day reuse
104 - if: steps.previous-cache-run.outputs.run_id != ''
140 + - name: Download previous snapshots for same-day reuse
141 + if: ${{ steps.previous-cache-run.outputs.run_id != '' && !inputs.rebuild_week }}
142 continue-on-error: true
143 uses: actions/download-artifact@v4
144 with:
145 github-token: ${{ secrets.GITHUB_TOKEN }}
146 run-id: ${{ steps.previous-cache-run.outputs.run_id }}
147 name: crawl-snapshots
111 - path: data/snapshots/
148 + path: .artifact-reuse/snapshots/
149
150 - name: Set up Python
151 uses: actions/setup-python@v5
@@ -119,22 +156,40 @@ jobs:
156 if: ${{ !inputs.rebuild_week }}
157 env:
158 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
122 - run: python scripts/crawl.py
159 + SOURCE_REFRESH_POLICY: ${{ steps.rerun-mode.outputs.source_refresh_policy }}
160 + run: |
161 + set -euo pipefail
162 + WEEK=$(date -u +%Y-W%V)
163 + CURRENT_DATETIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
164 + CODE_SHA=$(git hash-object scripts/crawl.py squadscope.topic.yml 2>/dev/null | sha256sum | cut -d' ' -f1)
165 + python scripts/crawl.py \
166 + --reuse-artifact ".artifact-reuse/raw-data/${WEEK}.json" \
167 + --source-refresh-policy "$SOURCE_REFRESH_POLICY" \
168 + --run-started-at "$CURRENT_DATETIME" \
169 + --current-code-sha "$CODE_SHA"
170
171 - name: Install Python dependencies
172 run: pip install -r requirements.txt
173
174 - name: Crawl external news RSS feeds
175 if: ${{ !inputs.rebuild_week }}
176 + env:
177 + SOURCE_REFRESH_POLICY: ${{ steps.rerun-mode.outputs.source_refresh_policy }}
178 run: |
130 - WEEK=$(date +%Y-W%V)
131 - SINCE=$(date -d '7 days ago' +%Y-%m-%d)
132 - UNTIL=$(date +%Y-%m-%d)
179 + WEEK=$(date -u +%Y-W%V)
180 + CURRENT_DATETIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
181 + SINCE=$(date -u -d '7 days ago' +%Y-%m-%d)
182 + UNTIL=$(date -u +%Y-%m-%d)
183 + CODE_SHA=$(git hash-object scripts/techcrunch_crawler.py config/external_news_sources.json 2>/dev/null | sha256sum | cut -d' ' -f1)
184 python scripts/techcrunch_crawler.py \
185 --sources config/external_news_sources.json \
186 --output "data/raw/${WEEK}-external-news.json" \
187 --since "$SINCE" \
137 - --until "$UNTIL"
188 + --until "$UNTIL" \
189 + --reuse-artifact ".artifact-reuse/raw-data/${WEEK}-external-news.json" \
190 + --source-refresh-policy "$SOURCE_REFRESH_POLICY" \
191 + --run-started-at "$CURRENT_DATETIME" \
192 + --current-code-sha "$CODE_SHA"
193 python3 - <<'PY' "data/raw/${WEEK}-external-news.json"
194 import json
195 import sys
@@ -178,7 +233,7 @@ jobs:
233 if-no-files-found: warn
234
235 - name: Commit crawl data to data branch
181 - if: ${{ !inputs.rebuild_week }}
236 + if: ${{ !inputs.rebuild_week && inputs.run_mode != 'dry-run' && inputs.run_mode != 'candidate-only' }}
237 env:
238 DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
239 DATA_BRANCH: publish
@@ -227,6 +282,7 @@ jobs:
282 candidate_summary_file: ${{ steps.analysis-context.outputs.candidate_output_file }}
283 publish_manifest_file: ${{ steps.analysis-context.outputs.publish_manifest_file }}
284 current_datetime: ${{ steps.analysis-context.outputs.current_datetime }}
285 + run_mode: ${{ steps.analysis-context.outputs.run_mode }}
286
287 steps:
288 - name: Check out repository
@@ -277,6 +333,8 @@ jobs:
333 id: analysis-context
334 env:
335 REBUILD_WEEK: ${{ inputs.rebuild_week || '' }}
336 + RUN_MODE: ${{ inputs.run_mode || 'normal' }}
337 + SOURCE_REFRESH_POLICY: ${{ inputs.source_refresh_policy || 'reuse-same-day' }}
338 run: |
339 mkdir -p data/analyzed data/candidates
340 CURRENT_DATETIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
@@ -314,6 +372,8 @@ jobs:
372 {
373 printf '%s\n' "${CONTEXT_LINES[@]}"
374 echo "current_datetime=$CURRENT_DATETIME"
375 + echo "run_mode=$RUN_MODE"
376 + echo "source_refresh_policy=$SOURCE_REFRESH_POLICY"
377 } >> "$GITHUB_OUTPUT"
378
379 - name: Install Python dependencies
@@ -619,6 +679,8 @@ jobs:
679 ANALYSIS_MODEL: ${{ steps.run-analysis.outputs.analysis_model }}
680 VALIDATION_STATUS: ${{ steps.quality-check.outcome == 'success' && 'passed' || 'failed' }}
681 MANIFEST_FILE: ${{ steps.analysis-context.outputs.publish_manifest_file }}
682 + RUN_MODE: ${{ steps.analysis-context.outputs.run_mode }}
683 + SOURCE_REFRESH_POLICY: ${{ steps.analysis-context.outputs.source_refresh_policy }}
684 GATE_REPORT: ${{ steps.analysis-context.outputs.analysis_gate_report_file }}
685 run: |
686 set -euo pipefail
@@ -643,14 +705,18 @@ jobs:
705 --analysis-source "$ANALYSIS_SOURCE" \
706 --analysis-model "$ANALYSIS_MODEL" \
707 --validation-status "$VALIDATION_STATUS" \
708 + --run-mode "$RUN_MODE" \
709 + --source-refresh-policy "$SOURCE_REFRESH_POLICY" \
710 --gate-report "$GATE_REPORT" \
711 --output "$MANIFEST_FILE" \
712 "${ARTIFACT_ARGS[@]}"
713
714 - name: Assert candidate is eligible for promotion
715 + if: ${{ steps.analysis-context.outputs.run_mode != 'dry-run' && steps.analysis-context.outputs.run_mode != 'candidate-only' }}
716 run: python3 scripts/publish_manifest.py assert-eligible --manifest "${{ steps.analysis-context.outputs.publish_manifest_file }}"
717
718 - name: Commit analysis and learnings to data branch
719 + if: ${{ steps.analysis-context.outputs.run_mode != 'dry-run' && steps.analysis-context.outputs.run_mode != 'candidate-only' }}
720 env:
721 DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
722 DATA_BRANCH: publish
@@ -729,6 +795,7 @@ jobs:
795
796 generate:
797 needs: analyze
798 + if: ${{ needs.analyze.outputs.run_mode != 'dry-run' && needs.analyze.outputs.run_mode != 'candidate-only' }}
799 runs-on: ubuntu-latest
800 permissions:
801 actions: read
@@ -869,6 +936,7 @@ jobs:
936
937 deploy:
938 needs: [crawl, analyze, generate]
939 + if: ${{ needs.analyze.outputs.run_mode != 'dry-run' && needs.analyze.outputs.run_mode != 'candidate-only' }}
940 runs-on: ubuntu-latest
941 permissions:
942 actions: read
@@ -946,7 +1014,7 @@ jobs:
1014 uses: actions/deploy-pages@v4
1015
1016 notify:
949 - if: github.event_name == 'schedule' || github.event.inputs.publish_release == 'true'
1017 + if: ${{ (github.event_name == 'schedule' || github.event.inputs.publish_release == 'true') && needs.analyze.outputs.run_mode != 'dry-run' && needs.analyze.outputs.run_mode != 'candidate-only' }}
1018 needs: [analyze, generate, deploy]
1019 runs-on: ubuntu-latest
1020 permissions:
docs/operator-guide.md
+14
@@ -142,6 +142,20 @@ Or through the GitHub UI:
142
143 The workflow takes ~2-3 minutes depending on GitHub API response times.
144
145 +#### Manual rerun modes
146 +
147 +Manual runs default to `run_mode=normal` and `source_refresh_policy=reuse-same-day`. Normal mode is fail-closed: it may publish only after the existing analysis and freshness gates pass, and same-day successful source artifacts are reused instead of scraping again. Missing, failed, stale, wrong-week, or wrong-window sources are refreshed.
148 +
149 +Use explicit modes for safer or destructive intent:
150 +
151 +- `dry-run`: build candidate artifacts only; never commit, deploy, notify, or publish a release.
152 +- `candidate-only`: run crawl/analysis and upload candidates; promotion is blocked by the manifest.
153 +- `restore`: requires `rebuild_week=YYYY-WNN`; hydrates artifacts from `publish` for audited restore/regeneration.
154 +- `force-replace`: explicit replacement intent, but gates still must pass before promotion.
155 +- `source_refresh_policy=force-refresh`: explicitly bypass same-day source reuse and refresh sources.
156 +
157 +Invalid combinations (for example `rebuild_week` without `run_mode=restore`, `publish_release` with `dry-run`, or `restore` with `force-refresh`) fail before publish content can be modified.
158 +
159 ### Option C: Run individual stages locally
160
161 For debugging or testing, run stages separately:
scripts/crawl.py
+116 -10
@@ -496,6 +496,27 @@ def parse_args() -> argparse.Namespace:
496 action="store_true",
497 help="Refresh GitHub data even when a same-day raw artifact is reusable.",
498 )
499 + parser.add_argument(
500 + "--reuse-artifact",
501 + default=None,
502 + help="Existing raw GitHub artifact to reuse when it is fresh for this run window.",
503 + )
504 + parser.add_argument(
505 + "--source-refresh-policy",
506 + choices=["reuse-same-day", "refresh-missing-stale", "force-refresh"],
507 + default="reuse-same-day",
508 + help="Source refresh policy for reruns (default: reuse eligible same-day artifacts).",
509 + )
510 + parser.add_argument(
511 + "--run-started-at",
512 + default=None,
513 + help="UTC run start timestamp used for same-day reuse checks (ISO 8601). Defaults to now.",
514 + )
515 + parser.add_argument(
516 + "--current-code-sha",
517 + default=None,
518 + help="Optional crawler/config fingerprint; reused artifacts with a conflicting fingerprint are stale.",
519 + )
520 return parser.parse_args()
521
522
@@ -605,6 +626,27 @@ def github_artifact_checksum(payload: dict[str, Any]) -> str:
626 return sha256_text(json.dumps(candidate, sort_keys=True, separators=(",", ":"), ensure_ascii=False))
627
628
629 +def parse_datetime(value: Any) -> datetime | None:
630 + if not isinstance(value, str) or not value.strip():
631 + return None
632 + candidate = value.strip()
633 + if candidate.endswith("Z"):
634 + candidate = f"{candidate[:-1]}+00:00"
635 + try:
636 + parsed = datetime.fromisoformat(candidate)
637 + except ValueError:
638 + return None
639 + return parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC)
640 +
641 +
642 +def load_json_artifact(path: Path) -> dict[str, Any] | None:
643 + try:
644 + payload = json.loads(path.read_text(encoding="utf-8"))
645 + except (OSError, json.JSONDecodeError):
646 + return None
647 + return payload if isinstance(payload, dict) else None
648 +
649 +
650 def load_reusable_github_payload(
651 path: Path,
652 *,
@@ -613,7 +655,11 @@ def load_reusable_github_payload(
655 since: datetime,
656 window_end: datetime,
657 config_checksum: str,
658 + policy: str = "reuse-same-day",
659 + current_code_sha: str | None = None,
660 ) -> dict[str, Any] | None:
661 + if policy == "force-refresh":
662 + return None
663 try:
664 payload = json.loads(path.read_text(encoding="utf-8"))
665 except (OSError, json.JSONDecodeError):
@@ -645,6 +691,10 @@ def load_reusable_github_payload(
691 or metadata.get("artifact_checksum") != github_artifact_checksum(payload)
692 ):
693 return None
694 + artifact_code_sha = metadata.get("crawler_code_sha")
695 + if current_code_sha and artifact_code_sha != current_code_sha:
696 + return None
697 + original_checksum = metadata.get("artifact_checksum")
698 metadata["same_day_reuse"] = {
699 "status": "reused",
700 "source": "github",
@@ -656,12 +706,50 @@ def load_reusable_github_payload(
706 "crawl_window": window,
707 "crawl_config_checksum": config_checksum,
708 "schema_checksum": github_schema_checksum(),
659 - "content_checksum": metadata.get("artifact_checksum"),
709 + "content_checksum": original_checksum,
710 }
711 + metadata["source_refresh_policy"] = policy
712 + if current_code_sha:
713 + metadata.setdefault("crawler_code_sha", current_code_sha)
714 payload["metadata"] = metadata
715 + metadata["artifact_checksum"] = github_artifact_checksum(payload)
716 return payload
717
718
719 +def _safe_snapshot_destination(snapshot_path: str, expected_snapshot_dir: Path = SNAPSHOT_ROOT) -> Path | None:
720 + destination = Path(snapshot_path)
721 + expected_root = Path("data") / "snapshots"
722 + if destination.is_absolute() or ".." in destination.parts:
723 + return None
724 + if len(destination.parts) < 3 or destination.parts[:2] != expected_root.parts:
725 + return None
726 + expected_dir = expected_snapshot_dir.resolve()
727 + resolved_destination = destination.resolve()
728 + if expected_dir != resolved_destination.parent and expected_dir not in resolved_destination.parents:
729 + return None
730 + return destination
731 +
732 +
733 +def restore_reused_snapshot(
734 + reuse_path: Path,
735 + metadata: dict[str, Any],
736 + *,
737 + expected_snapshot_dir: Path = SNAPSHOT_ROOT,
738 +) -> None:
739 + snapshot_path = metadata.get("snapshot_path")
740 + if not isinstance(snapshot_path, str) or not snapshot_path:
741 + return
742 + destination = _safe_snapshot_destination(snapshot_path, expected_snapshot_dir)
743 + if destination is None:
744 + return
745 + source_snapshot = reuse_path.parent.parent / "snapshots" / Path(snapshot_path).name
746 + if not source_snapshot.exists():
747 + return
748 + snapshot_payload = load_json_artifact(source_snapshot)
749 + if snapshot_payload is not None:
750 + write_payload(destination, snapshot_payload)
751 +
752 +
753 def load_previous_star_snapshot(snapshot_dir: Path, current_week: str, *raw_dirs: Path) -> dict[str, int]:
754 for snapshot in sorted(snapshot_dir.glob("*-stars.json"), reverse=True):
755 stars, reason = load_star_mapping_details(snapshot, current_week)
@@ -895,10 +983,6 @@ def write_payload(path: Path, payload: dict[str, Any]) -> None:
983
984 def main() -> int:
985 args = parse_args()
898 - github_token = os.environ.get("GITHUB_TOKEN")
899 - if not github_token:
900 - print("GITHUB_TOKEN is required", file=sys.stderr)
901 - return 1
986
987 topic_id = args.topic
988 topic_raw = raw_dir(topic_id)
@@ -906,29 +990,49 @@ def main() -> int:
990 topic_cache = cache_dir(topic_id)
991
992 crawled_at = utc_now()
993 + run_started_at_arg = getattr(args, "run_started_at", None)
994 + run_started_at = parse_datetime(run_started_at_arg) if run_started_at_arg else crawled_at
995 + if run_started_at is None:
996 + print("--run-started-at must be an ISO 8601 timestamp", file=sys.stderr)
997 + return 1
998 window_end = datetime.strptime(args.as_of, "%Y-%m-%d").replace(tzinfo=UTC) if args.as_of else crawled_at
999 since = datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=UTC) if args.since else window_end - timedelta(days=7)
1000 week = week_slug(window_end)
1001 output_path = Path(args.output) if args.output else topic_raw / f"{week}.json"
1002 snapshot_path = topic_snapshots / f"{week}-stars.json"
914 - client = GitHubClient(github_token, cache_dir=topic_cache)
1003 max_results = max(1, min(args.max_results, 1000))
1004 config_checksum = github_crawl_config_checksum(args, since, window_end, max_results)
1005 + source_refresh_policy = (
1006 + "force-refresh"
1007 + if getattr(args, "force_refresh", False)
1008 + else getattr(args, "source_refresh_policy", "reuse-same-day")
1009 + )
1010 + current_code_sha = getattr(args, "current_code_sha", None) or os.environ.get("CRAWLER_CODE_SHA")
1011
918 - if not getattr(args, "force_refresh", False):
1012 + if source_refresh_policy != "force-refresh":
1013 + reuse_path = Path(getattr(args, "reuse_artifact", "")) if getattr(args, "reuse_artifact", None) else output_path
1014 reusable = load_reusable_github_payload(
920 - output_path,
1015 + reuse_path,
1016 week=week,
922 - crawled_at=crawled_at,
1017 + crawled_at=run_started_at,
1018 since=since,
1019 window_end=window_end,
1020 config_checksum=config_checksum,
1021 + policy=source_refresh_policy,
1022 + current_code_sha=current_code_sha,
1023 )
1024 if reusable is not None:
1025 write_payload(output_path, reusable)
929 - print(f"Reused same-day GitHub raw artifact {output_path}; used 0 API calls.")
1026 + restore_reused_snapshot(reuse_path, reusable.get("metadata", {}), expected_snapshot_dir=topic_snapshots)
1027 + print(f"Reused same-day GitHub raw artifact {reuse_path} -> {output_path}; used 0 API calls.")
1028 return 0
1029
1030 + github_token = os.environ.get("GITHUB_TOKEN")
1031 + if not github_token:
1032 + print("GITHUB_TOKEN is required", file=sys.stderr)
1033 + return 1
1034 + client = GitHubClient(github_token, cache_dir=topic_cache)
1035 +
1036 if args.config:
1037 template_vars = {
1038 "last_week": since.date().isoformat(),
@@ -1004,6 +1108,8 @@ def main() -> int:
1108 "trending_repos": trending_filters,
1109 },
1110 "snapshot_path": snapshot_path.as_posix(),
1111 + "source_refresh_policy": source_refresh_policy,
1112 + "crawler_code_sha": current_code_sha or "",
1113 },
1114 }
1115 payload["metadata"]["artifact_checksum"] = github_artifact_checksum(payload)
scripts/publish_manifest.py
+33 -4
@@ -12,6 +12,8 @@ from typing import Any
12
13 SCHEMA_VERSION = "publish_eligibility_v1"
14 AI_SOURCES = {"copilot-cli", "github-models"}
15 +RUN_MODES = {"normal", "dry-run", "restore", "force-replace", "candidate-only"}
16 +SOURCE_REFRESH_POLICIES = {"reuse-same-day", "refresh-missing-stale", "force-refresh"}
17 ALLOWED_PROMOTION_MANIFEST_ROOTS = {("data", "staging"), ("data", "candidates")}
18 PROMOTION_MANIFEST_ROOT_ERROR = "Publish manifest must live under data/staging/ or data/candidates/."
19 NO_AI_SOURCE = "no-ai"
@@ -41,6 +43,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
43 create.add_argument("--analysis-source", required=True)
44 create.add_argument("--analysis-model", default="copilot-default")
45 create.add_argument("--validation-status", choices=["passed", "failed"], required=True)
46 + create.add_argument("--run-mode", choices=sorted(RUN_MODES), default="normal")
47 + create.add_argument("--source-refresh-policy", choices=sorted(SOURCE_REFRESH_POLICIES), default="reuse-same-day")
48 create.add_argument("--gate-report", type=Path, help="Structured analysis gate report emitted by analysis_gate.py.")
49 create.add_argument("--output", required=True, type=Path)
50 create.add_argument("--artifact", action="append", default=[], help="Additional source artifact as role=path.")
@@ -272,7 +276,7 @@ def same_day_reuse_status(payload: dict[str, Any] | None) -> dict[str, Any]:
276 }
277
278
275 -def freshness_for_json_artifact(role: str, week: str, payload: dict[str, Any] | None) -> dict[str, Any]:
279 +def freshness_for_json_artifact(role: str, week: str, payload: dict[str, Any] | None, *, run_date: datetime | None = None, run_mode: str = "normal") -> dict[str, Any]:
280 if payload is None:
281 return {"status": "missing" if role == "raw_github" else "not_applicable", "reasons": ["artifact missing"]}
282
@@ -288,6 +292,8 @@ def freshness_for_json_artifact(role: str, week: str, payload: dict[str, Any] |
292 reasons.append("missing or invalid crawled_at/generated_at timestamp")
293 elif week_slug(parsed) != week:
294 reasons.append(f"timestamp week mismatch: expected {week}, found {week_slug(parsed)}")
295 + elif run_date is not None and run_mode not in {"restore", "force-replace"} and parsed.astimezone(UTC).date() != run_date.astimezone(UTC).date():
296 + reasons.append("timestamp date is not the current UTC run date")
297
298 crawl_window = payload.get("crawl_window")
299 if role in {"external_news", "techcrunch_news"} and isinstance(crawl_window, dict):
@@ -298,7 +304,15 @@ def freshness_for_json_artifact(role: str, week: str, payload: dict[str, Any] |
304 return {"status": "fresh" if not reasons else "stale", "reasons": reasons}
305
306
301 -def artifact_entry(role: str, path: Path, week: str, generated_at: str | None = None) -> dict[str, Any]:
307 +def artifact_entry(
308 + role: str,
309 + path: Path,
310 + week: str,
311 + generated_at: str | None = None,
312 + *,
313 + run_date: datetime | None = None,
314 + run_mode: str = "normal",
315 +) -> dict[str, Any]:
316 payload = load_json(path) if path.suffix == ".json" else None
317 metadata = payload.get("metadata", {}) if isinstance(payload, dict) and isinstance(payload.get("metadata"), dict) else {}
318 if isinstance(payload, dict):
@@ -316,7 +330,7 @@ def artifact_entry(role: str, path: Path, week: str, generated_at: str | None =
330 "crawled_at": payload.get("crawled_at") if isinstance(payload, dict) else None,
331 "generated_at": artifact_generated_at,
332 "same_day_reuse": same_day_reuse_status(payload),
319 - "freshness": freshness_for_json_artifact(role, week, payload) if path.suffix == ".json" else {"status": "not_applicable", "reasons": []},
333 + "freshness": freshness_for_json_artifact(role, week, payload, run_date=run_date, run_mode=run_mode) if path.suffix == ".json" else {"status": "not_applicable", "reasons": []},
334 }
335 if "source_status" in metadata:
336 entry["source_status"] = metadata["source_status"]
@@ -423,7 +437,14 @@ def publishable_model_status(model: str) -> str:
437
438 def create_manifest(args: argparse.Namespace) -> int:
439 artifacts = [("raw_github", args.raw_json), *parse_artifacts(args.artifact)]
426 - source_artifacts = [artifact_entry(role, path, args.week, args.current_datetime) for role, path in artifacts if path.exists() or role == "raw_github"]
440 + run_date = parse_datetime(args.current_datetime)
441 + if run_date is None:
442 + raise SystemExit(f"Invalid --current-datetime value: {args.current_datetime!r}")
443 + source_artifacts = [
444 + artifact_entry(role, path, args.week, args.current_datetime, run_date=run_date, run_mode=args.run_mode)
445 + for role, path in artifacts
446 + if path.exists() or role == "raw_github"
447 + ]
448 artifact_reasons = [
449 f"{entry['role']}: {reason}"
450 for entry in source_artifacts
@@ -440,6 +461,7 @@ def create_manifest(args: argparse.Namespace) -> int:
461 candidate_content = args.content or args.summary
462 candidate_content_exists = candidate_content.exists()
463 validation_passed = args.validation_status == "passed"
464 + mode_allows_promotion = args.run_mode not in {"dry-run", "candidate-only"}
465 gates_passed = gate_report.get("present") is True and gate_report.get("passed") is True
466 candidate_quality = candidate_metadata.get("quality_score")
467 attempted_ai_paths = [path for path in args.attempted_ai_path if path.strip()]
@@ -493,6 +515,8 @@ def create_manifest(args: argparse.Namespace) -> int:
515 reasons.append("analysis validation did not pass")
516 if ai_status not in {"ai", "no-ai"}:
517 reasons.append(f"analysis source is not AI-publishable: {analysis_source or 'unknown'}")
518 + if not mode_allows_promotion:
519 + reasons.append(f"run mode {args.run_mode} is non-publishing")
520 if ai_status == "ai" and model_status != "available":
521 reasons.append(f"analysis model is not AI-publishable: {args.analysis_model or 'unknown'}")
522 if not gates_passed:
@@ -507,6 +531,7 @@ def create_manifest(args: argparse.Namespace) -> int:
531 and gates_passed
532 and not artifact_reasons
533 and not comparison_reasons
534 + and mode_allows_promotion
535 and not reasons
536 and (
537 (ai_status == "ai" and model_status == "available")
@@ -521,6 +546,8 @@ def create_manifest(args: argparse.Namespace) -> int:
546 "run_id": args.run_id,
547 "week": args.week,
548 "generated_at": args.current_datetime,
549 + "run_mode": args.run_mode,
550 + "source_refresh_policy": args.source_refresh_policy,
551 "run_started_at": args.current_datetime,
552 "candidate_summary_path": args.summary.as_posix(),
553 "candidate_content_path": candidate_content.as_posix(),
@@ -591,6 +618,8 @@ def create_manifest(args: argparse.Namespace) -> int:
618 "promotion": {
619 "eligible": eligible,
620 "decision": decision,
621 + "mode": args.run_mode,
622 + "source_refresh_policy": args.source_refresh_policy,
623 "policy": args.publish_policy,
624 "reasons": reasons,
625 },
scripts/rerun_modes.py new
+101
@@ -0,0 +1,101 @@
1 +#!/usr/bin/env python3
2 +"""Validate crawl-and-publish rerun modes before any publishing side effects."""
3 +
4 +from __future__ import annotations
5 +
6 +import argparse
7 +import json
8 +from dataclasses import dataclass
9 +from pathlib import Path
10 +
11 +RUN_MODES = {"normal", "dry-run", "restore", "force-replace", "candidate-only"}
12 +SOURCE_REFRESH_POLICIES = {"reuse-same-day", "refresh-missing-stale", "force-refresh"}
13 +
14 +
15 +@dataclass(frozen=True)
16 +class ModeDecision:
17 + run_mode: str
18 + source_refresh_policy: str
19 + action: str
20 + publish_allowed: bool
21 + crawl_allowed: bool
22 + reasons: list[str]
23 +
24 +
25 +def validate_modes(
26 + *,
27 + run_mode: str,
28 + source_refresh_policy: str,
29 + rebuild_week: str = "",
30 + publish_release: bool = False,
31 +) -> ModeDecision:
32 + reasons: list[str] = []
33 + if run_mode not in RUN_MODES:
34 + reasons.append(f"invalid run_mode: {run_mode}")
35 + if source_refresh_policy not in SOURCE_REFRESH_POLICIES:
36 + reasons.append(f"invalid source_refresh_policy: {source_refresh_policy}")
37 + if rebuild_week and run_mode != "restore":
38 + reasons.append("rebuild_week is a restore operation and requires run_mode=restore")
39 + if run_mode == "restore" and not rebuild_week:
40 + reasons.append("run_mode=restore requires rebuild_week=YYYY-WNN")
41 + if run_mode in {"dry-run", "candidate-only"} and publish_release:
42 + reasons.append(f"publish_release is not allowed with run_mode={run_mode}")
43 + if run_mode == "restore" and source_refresh_policy == "force-refresh":
44 + reasons.append("run_mode=restore hydrates publish artifacts and cannot force-refresh sources")
45 +
46 + publish_allowed = run_mode not in {"dry-run", "candidate-only"}
47 + crawl_allowed = not rebuild_week
48 + if run_mode == "dry-run":
49 + action = "analyze candidate only; never commit or deploy"
50 + elif run_mode == "candidate-only":
51 + action = "produce candidate artifacts only; never promote"
52 + elif run_mode == "restore":
53 + action = f"restore published artifacts for {rebuild_week} and regenerate through guarded promotion"
54 + elif run_mode == "force-replace":
55 + action = "explicit replacement run; promotion still requires all gates"
56 + else:
57 + action = "normal guarded crawl, analysis, publish, and deploy"
58 +
59 + return ModeDecision(run_mode, source_refresh_policy, action, publish_allowed, crawl_allowed, reasons)
60 +
61 +
62 +def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
63 + parser = argparse.ArgumentParser(description=__doc__)
64 + parser.add_argument("--run-mode", default="normal")
65 + parser.add_argument("--source-refresh-policy", default="reuse-same-day")
66 + parser.add_argument("--rebuild-week", default="")
67 + parser.add_argument("--publish-release", action="store_true")
68 + parser.add_argument("--summary-json", type=Path)
69 + return parser.parse_args(argv)
70 +
71 +
72 +def main(argv: list[str] | None = None) -> int:
73 + args = parse_args(argv)
74 + decision = validate_modes(
75 + run_mode=args.run_mode,
76 + source_refresh_policy=args.source_refresh_policy,
77 + rebuild_week=args.rebuild_week.strip(),
78 + publish_release=args.publish_release,
79 + )
80 + payload = {
81 + "run_mode": decision.run_mode,
82 + "source_refresh_policy": decision.source_refresh_policy,
83 + "publish_allowed": decision.publish_allowed,
84 + "crawl_allowed": decision.crawl_allowed,
85 + "action": decision.action,
86 + "valid": not decision.reasons,
87 + "reasons": decision.reasons,
88 + }
89 + if args.summary_json:
90 + args.summary_json.parent.mkdir(parents=True, exist_ok=True)
91 + args.summary_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
92 + print(json.dumps(payload, sort_keys=True))
93 + if decision.reasons:
94 + for reason in decision.reasons:
95 + print(f"::error::{reason}")
96 + return 1
97 + return 0
98 +
99 +
100 +if __name__ == "__main__":
101 + raise SystemExit(main())
scripts/techcrunch_crawler.py
+138 -5
@@ -228,6 +228,88 @@ def _same_window(payload: dict[str, Any], since: datetime, until: datetime) -> b
228 return isinstance(window, dict) and window.get("since") == iso_timestamp(since) and window.get("until") == iso_timestamp(until)
229
230
231 +def same_utc_day(value: str | None, expected: date) -> bool:
232 + parsed = parse_iso_datetime(value)
233 + return parsed is not None and parsed.astimezone(UTC).date() == expected
234 +
235 +
236 +def source_reuse_decisions(
237 + payload: dict[str, Any] | None,
238 + sources: list[NewsSourceConfig],
239 + *,
240 + week: str,
241 + run_date: date,
242 + since: datetime,
243 + until: datetime,
244 + policy: str,
245 + current_config_checksum: str,
246 + current_code_sha: str | None,
247 +) -> tuple[list[dict[str, Any]], list[NewsSourceConfig], list[dict[str, Any]], list[dict[str, Any]]]:
248 + """Compatibility planner for callers that pass a loaded artifact."""
249 + decisions: list[dict[str, Any]] = []
250 + reused_articles: list[dict[str, Any]] = []
251 + reused_statuses: list[dict[str, Any]] = []
252 + to_crawl: list[NewsSourceConfig] = []
253 + metadata = payload.get("metadata", {}) if isinstance(payload, dict) else {}
254 + statuses = metadata.get("source_status", []) if isinstance(metadata, dict) else []
255 + if not isinstance(statuses, list):
256 + statuses = []
257 + status_by_source = {str(status.get("source")): status for status in statuses if isinstance(status, dict)}
258 + raw_articles = payload.get("articles", []) if isinstance(payload, dict) else []
259 + articles: list[dict[str, Any]] = []
260 + articles_malformed = False
261 + if isinstance(raw_articles, list):
262 + for article in raw_articles:
263 + article_sources = article.get("sources", []) if isinstance(article, dict) else None
264 + if not isinstance(article, dict) or not isinstance(article_sources, list):
265 + articles_malformed = True
266 + break
267 + articles.append(article)
268 + else:
269 + articles_malformed = True
270 + global_reasons: list[str] = []
271 + if policy == "force-refresh":
272 + global_reasons.append("source_refresh_policy=force-refresh")
273 + if payload is None:
274 + global_reasons.append("artifact missing or malformed")
275 + else:
276 + if payload.get("week") != week:
277 + global_reasons.append(f"week mismatch: expected {week}, found {payload.get('week')!r}")
278 + if not same_utc_day(payload.get("crawled_at"), run_date):
279 + global_reasons.append("artifact is not from the current UTC run date")
280 + window = payload.get("crawl_window") if isinstance(payload.get("crawl_window"), dict) else {}
281 + if window.get("since") != iso_timestamp(since) or window.get("until") != iso_timestamp(until):
282 + global_reasons.append("crawl window mismatch")
283 + if isinstance(metadata, dict):
284 + if metadata.get("source_config_checksum") != current_config_checksum:
285 + global_reasons.append("source config checksum mismatch")
286 + artifact_code_sha = metadata.get("crawler_code_sha")
287 + if current_code_sha and artifact_code_sha != current_code_sha:
288 + global_reasons.append("crawler/config fingerprint mismatch")
289 + if articles_malformed:
290 + global_reasons.append("artifact articles malformed")
291 + for source in sources:
292 + source_reasons = list(global_reasons)
293 + status = status_by_source.get(source.name)
294 + if not status or status.get("success") is not True:
295 + source_reasons.append("source missing or previously failed")
296 + if source_reasons:
297 + decisions.append({"source": source.name, "decision": "refresh", "reasons": source_reasons})
298 + to_crawl.append(source)
299 + continue
300 + source_articles = [
301 + article for article in articles
302 + if source.name in {str(article.get("source", "")), *[str(item) for item in article.get("sources", [])]}
303 + ]
304 + reused_articles.extend(source_articles)
305 + reused_status = dict(status)
306 + reused_status["reused_same_day"] = True
307 + reused_status["success"] = True
308 + reused_statuses.append(reused_status)
309 + decisions.append({"source": source.name, "decision": "reuse", "reasons": []})
310 + return reused_articles, to_crawl, reused_statuses, decisions
311 +
312 +
313 def plan_source_reuse(
314 previous_path: Path,
315 sources: list[NewsSourceConfig],
@@ -237,9 +319,13 @@ def plan_source_reuse(
319 until: datetime,
320 config_checksum: str,
321 forced_sources: set[str] | None = None,
322 + source_refresh_policy: str = "reuse-same-day",
323 + run_started_at: datetime | None = None,
324 + current_code_sha: str | None = None,
325 ) -> tuple[list[dict[str, Any]], list[NewsSourceConfig], list[dict[str, Any]], list[dict[str, Any]], list[dict[str, str]]]:
326 """Load eligible same-day source artifacts and return reused articles plus sources to crawl."""
327 forced = forced_sources or set()
328 + run_time = run_started_at or now
329 requested = {source.name for source in sources}
330 pending: list[NewsSourceConfig] = []
331 reused_articles: list[dict[str, Any]] = []
@@ -249,7 +335,9 @@ def plan_source_reuse(
335 previous = _load_json_object(previous_path) if previous_path.exists() else None
336 expected_schema_checksum = schema_checksum()
337
252 - if previous is None:
338 + if source_refresh_policy == "force-refresh":
339 + stale_reasons = ["source_refresh_policy=force-refresh"]
340 + elif previous is None:
341 stale_reasons = ["missing previous artifact" if not previous_path.exists() else "previous artifact is not valid JSON"]
342 else:
343 crawled_at = parse_iso_datetime(previous.get("crawled_at"))
@@ -260,7 +348,7 @@ def plan_source_reuse(
348 stale_reasons.append(str(exc))
349 if previous.get("week") != week_slug(now):
350 stale_reasons.append(f"week mismatch: expected {week_slug(now)}, found {previous.get('week')!r}")
263 - if crawled_at is None or crawled_at.astimezone(UTC).date() != now.astimezone(UTC).date():
351 + if crawled_at is None or crawled_at.astimezone(UTC).date() != run_time.astimezone(UTC).date():
352 stale_reasons.append("crawled_at is not from the current UTC day")
353 if not _same_window(previous, since, until):
354 stale_reasons.append("crawl_window mismatch")
@@ -268,6 +356,9 @@ def plan_source_reuse(
356 stale_reasons.append("source_config_checksum mismatch")
357 if metadata.get("schema_checksum") != expected_schema_checksum:
358 stale_reasons.append("schema_checksum mismatch")
359 + artifact_code_sha = metadata.get("crawler_code_sha")
360 + if current_code_sha and artifact_code_sha != current_code_sha:
361 + stale_reasons.append("crawler/config fingerprint mismatch")
362
363 previous_metadata = previous.get("metadata", {}) if isinstance(previous, dict) and isinstance(previous.get("metadata"), dict) else {}
364 previous_statuses = {
@@ -919,9 +1010,34 @@ def main(argv: list[str] | None = None) -> int:
1010 default=[],
1011 help="Refresh one source by id even when its same-day artifact is reusable. Can be repeated.",
1012 )
1013 + parser.add_argument(
1014 + "--reuse-artifact",
1015 + default=None,
1016 + help="Existing external-news artifact to reuse per source when fresh for this run window.",
1017 + )
1018 + parser.add_argument(
1019 + "--source-refresh-policy",
1020 + choices=["reuse-same-day", "refresh-missing-stale", "force-refresh"],
1021 + default="reuse-same-day",
1022 + help="Source refresh policy for reruns (default: reuse eligible same-day sources).",
1023 + )
1024 + parser.add_argument(
1025 + "--run-started-at",
1026 + default=None,
1027 + help="UTC run start timestamp used for same-day reuse checks (ISO 8601). Defaults to now.",
1028 + )
1029 + parser.add_argument(
1030 + "--current-code-sha",
1031 + default=None,
1032 + help="Optional crawler/config fingerprint; reused artifacts with a conflicting fingerprint are stale.",
1033 + )
1034 args = parser.parse_args(argv)
1035
1036 now = datetime.now(UTC)
1037 + run_started_at = parse_iso_datetime(args.run_started_at) if args.run_started_at else now
1038 + if run_started_at is None:
1039 + print("--run-started-at must be an ISO 8601 timestamp", file=sys.stderr)
1040 + return 1
1041 since = (
1042 datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=UTC)
1043 if args.since
@@ -942,15 +1058,21 @@ def main(argv: list[str] | None = None) -> int:
1058 out_path = out_dir / f"{week_slug(now)}-external-news.json"
1059
1060 config_checksum = source_config_checksum(source_configs)
945 - force_sources = {source.name for source in source_configs} if args.force_refresh else set(args.force_refresh_source or [])
1061 + source_refresh_policy = "force-refresh" if args.force_refresh else args.source_refresh_policy
1062 + current_code_sha = args.current_code_sha or ""
1063 + force_sources = {source.name for source in source_configs} if source_refresh_policy == "force-refresh" else set(args.force_refresh_source or [])
1064 + reuse_path = Path(args.reuse_artifact) if args.reuse_artifact else out_path
1065 reused_articles, sources_to_crawl, reuse_summary, provenance, _ = plan_source_reuse(
947 - out_path,
1066 + reuse_path,
1067 source_configs,
1068 now=now,
1069 since=since,
1070 until=until,
1071 config_checksum=config_checksum,
1072 forced_sources=force_sources,
1073 + source_refresh_policy=source_refresh_policy,
1074 + run_started_at=run_started_at,
1075 + current_code_sha=current_code_sha,
1076 )
1077 refreshed_articles, errors, refreshed_statuses = crawl_sources_parallel(
1078 sources_to_crawl, since=since, until=until, max_workers=args.max_workers
@@ -969,7 +1091,7 @@ def main(argv: list[str] | None = None) -> int:
1091 run_id=os.environ.get("GITHUB_RUN_ID", "local"),
1092 )
1093 reused_sources = {entry["source_id"] for entry in provenance if entry.get("action") == "reused"}
972 - previous = _load_json_object(out_path) if out_path.exists() else None
1094 + previous = _load_json_object(reuse_path) if reuse_path.exists() else None
1095 previous_metadata = previous.get("metadata", {}) if isinstance(previous, dict) and isinstance(previous.get("metadata"), dict) else {}
1096 previous_statuses = [
1097 {**status, "reused": True}
@@ -994,6 +1116,17 @@ def main(argv: list[str] | None = None) -> int:
1116 source_artifact_provenance=provenance,
1117 run_id=os.environ.get("GITHUB_RUN_ID", "local"),
1118 )
1119 + output["metadata"]["same_day_reuse"] = (
1120 + "mixed" if reused_articles and refreshed_articles else "reused" if reused_articles else "not_reused"
1121 + )
1122 + output["metadata"]["source_refresh_policy"] = source_refresh_policy
1123 + output["metadata"]["source_reuse_decisions"] = [
1124 + {"source": item["source"], "decision": "reuse" if item["action"] == "reused" else "refresh", "reasons": item["reasons"]}
1125 + for item in output["metadata"]["source_reuse_summary"]
1126 + ]
1127 + output["metadata"]["crawler_code_sha"] = current_code_sha
1128 + output["metadata"]["artifact_checksum"] = artifact_checksum(output)
1129 + validate_canonical_output(output)
1130
1131 out_path.parent.mkdir(parents=True, exist_ok=True)
1132 with open(out_path, "w", encoding="utf-8") as f:
tests/test_crawl.py
+119
@@ -218,6 +218,64 @@ class CrawlTests(unittest.TestCase):
218 self.assertEqual(reused["metadata"]["same_day_reuse"]["status"], "reused")
219 self.assertEqual(reused["metadata"]["same_day_reuse"]["source_id"], "github-search")
220
221 + def test_main_reuses_valid_same_day_raw_artifact_without_github_token(self) -> None:
222 + tests_root = Path(__file__).resolve().parent
223 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
224 + base = Path(tmpdir)
225 + existing = base / "reuse/raw-data/2026-W21.json"
226 + output = base / "data/raw/2026-W21.json"
227 + args = Namespace(
228 + since="2026-05-12",
229 + as_of="2026-05-19",
230 + max_results=25,
231 + output=str(output),
232 + topic=None,
233 + config=None,
234 + reuse_artifact=str(existing),
235 + source_refresh_policy="reuse-same-day",
236 + run_started_at="2026-05-19T10:00:00Z",
237 + current_code_sha="sha",
238 + )
239 + since = datetime(2026, 5, 12, tzinfo=crawl.UTC)
240 + window_end = datetime(2026, 5, 19, tzinfo=crawl.UTC)
241 + checksum = crawl.github_crawl_config_checksum(args, since, window_end, 25)
242 + payload = {
243 + "week": "2026-W21",
244 + "crawled_at": "2026-05-19T08:00:00Z",
245 + "new_repos": [],
246 + "trending_repos": [],
247 + "signals": {"top_topics": []},
248 + "metadata": {
249 + "api_calls_used": 1,
250 + "cache_hits": 0,
251 + "stale_cache_hits": 0,
252 + "rate_limit_limit": None,
253 + "rate_limit_remaining": None,
254 + "rate_limit_reset": None,
255 + "rate_limit_resource": None,
256 + "partial_failures": [],
257 + "run_id": "111111",
258 + "snapshot_path": "data/snapshots/2026-W21-stars.json",
259 + "crawl_window": {"since": "2026-05-12", "until": "2026-05-19"},
260 + "crawl_config_checksum": checksum,
261 + "schema_checksum": crawl.github_schema_checksum(),
262 + "same_day_reuse": {"status": "not_reused", "source": "github", "source_id": crawl.GITHUB_SOURCE_ID},
263 + "crawler_code_sha": "sha",
264 + },
265 + }
266 + payload["metadata"]["artifact_checksum"] = crawl.github_artifact_checksum(payload)
267 + crawl.write_payload(existing, payload)
268 +
269 + with mock.patch.object(crawl, "parse_args", return_value=args), mock.patch.dict(
270 + "os.environ", {}, clear=True
271 + ), mock.patch.object(crawl, "utc_now", return_value=datetime(2026, 5, 19, 10, 0, tzinfo=crawl.UTC)):
272 + exit_code = crawl.main()
273 +
274 + self.assertEqual(exit_code, 0)
275 + reused = json.loads(output.read_text(encoding="utf-8"))
276 + self.assertEqual(reused["metadata"]["same_day_reuse"]["status"], "reused")
277 + self.assertEqual(reused["metadata"]["source_refresh_policy"], "reuse-same-day")
278 +
279 def test_main_emits_github_source_id_in_same_day_reuse_metadata(self) -> None:
280 class FakeClient:
281 def __init__(self, token: str, **kwargs) -> None:
@@ -308,6 +366,67 @@ class CrawlTests(unittest.TestCase):
366
367 self.assertIsNone(reused)
368
369 + def test_load_reusable_github_payload_rejects_missing_code_fingerprint_when_required(self) -> None:
370 + tests_root = Path(__file__).resolve().parent
371 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
372 + base = Path(tmpdir)
373 + output = base / "data/raw/2026-W21.json"
374 + args = Namespace(since="2026-05-12", as_of="2026-05-19", max_results=25, output=str(output), topic=None, config=None)
375 + since = datetime(2026, 5, 12, tzinfo=crawl.UTC)
376 + window_end = datetime(2026, 5, 19, tzinfo=crawl.UTC)
377 + checksum = crawl.github_crawl_config_checksum(args, since, window_end, 25)
378 + payload = {
379 + "week": "2026-W21",
380 + "crawled_at": "2026-05-19T08:00:00Z",
381 + "new_repos": [],
382 + "trending_repos": [],
383 + "signals": {"top_topics": []},
384 + "metadata": {
385 + "api_calls_used": 1,
386 + "cache_hits": 0,
387 + "stale_cache_hits": 0,
388 + "rate_limit_limit": None,
389 + "rate_limit_remaining": None,
390 + "rate_limit_reset": None,
391 + "rate_limit_resource": None,
392 + "partial_failures": [],
393 + "snapshot_path": "data/snapshots/2026-W21-stars.json",
394 + "crawl_window": {"since": "2026-05-12", "until": "2026-05-19"},
395 + "crawl_config_checksum": checksum,
396 + "schema_checksum": crawl.github_schema_checksum(),
397 + "same_day_reuse": {"status": "not_reused", "source": "github"},
398 + },
399 + }
400 + payload["metadata"]["artifact_checksum"] = crawl.github_artifact_checksum(payload)
401 + crawl.write_payload(output, payload)
402 +
403 + reused = crawl.load_reusable_github_payload(
404 + output,
405 + week="2026-W21",
406 + crawled_at=datetime(2026, 5, 19, 10, 0, tzinfo=crawl.UTC),
407 + since=since,
408 + window_end=window_end,
409 + config_checksum=checksum,
410 + current_code_sha="sha",
411 + )
412 +
413 + self.assertIsNone(reused)
414 +
415 + def test_restore_reused_snapshot_rejects_unsafe_metadata_path(self) -> None:
416 + tests_root = Path(__file__).resolve().parent
417 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
418 + base = Path(tmpdir)
419 + reuse_path = base / "reuse/raw/2026-W21.json"
420 + source_snapshot = base / "reuse/snapshots/2026-W21-stars.json"
421 + source_snapshot.parent.mkdir(parents=True)
422 + source_snapshot.write_text('{"stars": {"owner/repo": 1}}\n', encoding="utf-8")
423 +
424 + for unsafe_path in ("/home/azureuser/source/SquadScope/data/snapshots/2026-W21-stars.json", "data/snapshots/../raw/evil.json"):
425 + with mock.patch.object(crawl, "write_payload") as write_mock:
426 + crawl.restore_reused_snapshot(reuse_path, {"snapshot_path": unsafe_path})
427 +
428 + write_mock.assert_not_called()
429 +
430
431 if __name__ == "__main__":
432 unittest.main()
tests/test_pipeline.py
+25 -2
@@ -188,7 +188,8 @@ class WorkflowConfigTests(unittest.TestCase):
188
189 self.assertIsNotNone(external_news_step, "External news crawl step not found")
190 run_script = external_news_step["run"]
191 - self.assertIn("UNTIL=$(date +%Y-%m-%d)", run_script)
191 + self.assertIn("SINCE=$(date -u -d '7 days ago' +%Y-%m-%d)", run_script)
192 + self.assertIn("UNTIL=$(date -u +%Y-%m-%d)", run_script)
193 self.assertIn('--until "$UNTIL"', run_script)
194
195 def test_crawl_workflow_defines_reskill_jobs(self) -> None:
@@ -385,8 +386,10 @@ class WorkflowConfigTests(unittest.TestCase):
386 self.assertIn("scripts/publish_manifest.py create", manifest_run)
387 self.assertIn("--analysis-source", manifest_run)
388 self.assertIn("--analysis-model", manifest_run)
388 - self.assertIn('git checkout origin/publish -- "$PUBLISHED_SUMMARY"', manifest_run)
389 self.assertIn('--validation-status "$VALIDATION_STATUS"', manifest_run)
390 + self.assertIn("--run-mode", manifest_run)
391 + self.assertIn("--source-refresh-policy", manifest_run)
392 + self.assertIn('git checkout origin/publish -- "$PUBLISHED_SUMMARY"', manifest_run)
393
394 assert_step = next((s for s in analyze["steps"] if s.get("name") == "Assert candidate is eligible for promotion"), None)
395 self.assertIsNotNone(assert_step)
@@ -408,6 +411,26 @@ class WorkflowConfigTests(unittest.TestCase):
411 self.assertIsNotNone(generate_step)
412 self.assertIn('assert-eligible --manifest "$MANIFEST_FILE"', generate_step["run"])
413
414 + def test_rerun_mode_inputs_and_guards_are_declared(self) -> None:
415 + workflow_path = Path(".github/workflows/crawl-and-publish.yml")
416 + workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8"))
417 + inputs = workflow[True]["workflow_dispatch"]["inputs"]
418 +
419 + self.assertEqual(inputs["run_mode"]["default"], "normal")
420 + self.assertIn("restore", inputs["run_mode"]["options"])
421 + self.assertEqual(inputs["source_refresh_policy"]["default"], "reuse-same-day")
422 + self.assertIn("force-refresh", inputs["source_refresh_policy"]["options"])
423 +
424 + crawl_steps = workflow["jobs"]["crawl"]["steps"]
425 + validate_step = next((s for s in crawl_steps if s.get("name") == "Validate rerun mode"), None)
426 + self.assertIsNotNone(validate_step)
427 + self.assertIn("scripts/rerun_modes.py", validate_step["run"])
428 +
429 + run_crawler = next((s for s in crawl_steps if s.get("name") == "Run crawler"), None)
430 + self.assertIn("--reuse-artifact", run_crawler["run"])
431 + self.assertIn("--source-refresh-policy", run_crawler["run"])
432 +
433 +
434 def test_notify_failure_job_creates_or_updates_issue(self) -> None:
435 workflow_path = Path(".github/workflows/crawl-and-publish.yml")
436 workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8"))
tests/test_publish_manifest.py
+121
@@ -648,6 +648,127 @@ class PublishManifestTests(unittest.TestCase):
648 self.assertEqual(reuse["status"], "reused")
649 self.assertEqual(reuse["source_id"], "github-search")
650
651 + def test_same_week_wrong_day_source_blocks_normal_promotion(self) -> None:
652 + tests_root = Path(__file__).resolve().parent
653 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
654 + base = Path(tmpdir)
655 + raw = base / "data/raw/2026-W21.json"
656 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
657 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
658 + write_raw(raw, crawled_at="2026-05-19T08:00:00Z")
659 + write_summary(summary)
660 +
661 + publish_manifest.main(
662 + [
663 + "create",
664 + "--week",
665 + WEEK,
666 + "--run-id",
667 + RUN_ID,
668 + "--current-datetime",
669 + "2026-05-20T08:00:00Z",
670 + "--summary",
671 + str(summary),
672 + "--published-summary",
673 + str(base / "data/analyzed/2026-W21-summary.md"),
674 + "--raw-json",
675 + str(raw),
676 + "--analysis-source",
677 + "copilot-cli",
678 + "--analysis-model",
679 + "copilot-default",
680 + "--validation-status",
681 + "passed",
682 + "--output",
683 + str(manifest),
684 + ]
685 + )
686 +
687 + payload = json.loads(manifest.read_text(encoding="utf-8"))
688 + self.assertFalse(payload["promotion"]["eligible"])
689 + self.assertTrue(any("current UTC run date" in reason for reason in payload["promotion"]["reasons"]))
690 +
691 + def test_invalid_current_datetime_fails_manifest_creation(self) -> None:
692 + tests_root = Path(__file__).resolve().parent
693 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
694 + base = Path(tmpdir)
695 + raw = base / "data/raw/2026-W21.json"
696 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
697 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
698 + write_raw(raw)
699 + write_summary(summary)
700 +
701 + with self.assertRaises(SystemExit):
702 + publish_manifest.main(
703 + [
704 + "create",
705 + "--week",
706 + WEEK,
707 + "--run-id",
708 + RUN_ID,
709 + "--current-datetime",
710 + "not-a-date",
711 + "--summary",
712 + str(summary),
713 + "--published-summary",
714 + str(base / "data/analyzed/2026-W21-summary.md"),
715 + "--raw-json",
716 + str(raw),
717 + "--analysis-source",
718 + "copilot-cli",
719 + "--analysis-model",
720 + "copilot-default",
721 + "--validation-status",
722 + "passed",
723 + "--output",
724 + str(manifest),
725 + ]
726 + )
727 + self.assertFalse(manifest.exists())
728 +
729 + def test_candidate_only_mode_blocks_promotion_even_with_valid_sources(self) -> None:
730 + tests_root = Path(__file__).resolve().parent
731 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
732 + base = Path(tmpdir)
733 + raw = base / "data/raw/2026-W21.json"
734 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
735 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
736 + write_raw(raw)
737 + write_summary(summary)
738 +
739 + publish_manifest.main(
740 + [
741 + "create",
742 + "--week",
743 + WEEK,
744 + "--run-id",
745 + RUN_ID,
746 + "--current-datetime",
747 + CURRENT_DATETIME,
748 + "--summary",
749 + str(summary),
750 + "--published-summary",
751 + str(base / "data/analyzed/2026-W21-summary.md"),
752 + "--raw-json",
753 + str(raw),
754 + "--analysis-source",
755 + "copilot-cli",
756 + "--analysis-model",
757 + "copilot-default",
758 + "--validation-status",
759 + "passed",
760 + "--run-mode",
761 + "candidate-only",
762 + "--output",
763 + str(manifest),
764 + ]
765 + )
766 +
767 + payload = json.loads(manifest.read_text(encoding="utf-8"))
768 + self.assertFalse(payload["promotion"]["eligible"])
769 + self.assertEqual(payload["run_mode"], "candidate-only")
770 + self.assertTrue(any("non-publishing" in reason for reason in payload["promotion"]["reasons"]))
771 +
772 def test_failed_gate_report_blocks_promotion(self) -> None:
773 tests_root = Path(__file__).resolve().parent
774 with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
tests/test_rerun_modes.py new
+32
@@ -0,0 +1,32 @@
1 +import unittest
2 +
3 +from scripts.rerun_modes import validate_modes
4 +
5 +
6 +class RerunModeTests(unittest.TestCase):
7 + def test_normal_mode_allows_guarded_publish_and_reuse_default(self) -> None:
8 + decision = validate_modes(run_mode="normal", source_refresh_policy="reuse-same-day")
9 +
10 + self.assertFalse(decision.reasons)
11 + self.assertTrue(decision.publish_allowed)
12 + self.assertTrue(decision.crawl_allowed)
13 + self.assertIn("normal guarded", decision.action)
14 +
15 + def test_rebuild_week_requires_explicit_restore_mode(self) -> None:
16 + decision = validate_modes(
17 + run_mode="normal",
18 + source_refresh_policy="reuse-same-day",
19 + rebuild_week="2026-W21",
20 + )
21 +
22 + self.assertIn("requires run_mode=restore", decision.reasons[0])
23 +
24 + def test_dry_run_cannot_publish_release(self) -> None:
25 + decision = validate_modes(
26 + run_mode="dry-run",
27 + source_refresh_policy="reuse-same-day",
28 + publish_release=True,
29 + )
30 +
31 + self.assertFalse(decision.publish_allowed)
32 + self.assertTrue(any("publish_release" in reason for reason in decision.reasons))
tests/test_techcrunch_crawler.py
+134
@@ -26,6 +26,8 @@ from scripts.techcrunch_crawler import (
26 iso_timestamp,
27 load_source_configs,
28 parse_published_date,
29 + source_config_checksum,
30 + source_reuse_decisions,
31 validate_feed_url,
32 week_slug,
33 )
@@ -314,6 +316,116 @@ class TestExternalNewsSources:
316 assert "mit_technology_review" in names
317 assert "github_blog" in names
318
319 +
320 + def test_source_reuse_decisions_reuses_successful_same_day_sources_and_refreshes_failed(self):
321 + sources = [
322 + NewsSourceConfig("techcrunch", "https://techcrunch.com/feed/"),
323 + NewsSourceConfig("github-blog", "https://github.blog/feed/"),
324 + ]
325 + since = datetime(2026, 5, 11, tzinfo=UTC)
326 + until = datetime(2026, 5, 18, tzinfo=UTC)
327 + payload = {
328 + "week": "2026-W21",
329 + "crawled_at": "2026-05-18T08:00:00Z",
330 + "crawl_window": {"since": iso_timestamp(since), "until": iso_timestamp(until)},
331 + "articles": [{"source": "techcrunch", "title": "Reused", "published_at": "2026-05-17T00:00:00Z"}],
332 + "metadata": {
333 + "source_config_checksum": source_config_checksum(sources),
334 + "source_status": [
335 + {"source": "techcrunch", "success": True},
336 + {"source": "github-blog", "success": False},
337 + ],
338 + },
339 + }
340 +
341 + reused, to_crawl, reused_statuses, decisions = source_reuse_decisions(
342 + payload,
343 + sources,
344 + week="2026-W21",
345 + run_date=datetime(2026, 5, 18, tzinfo=UTC).date(),
346 + since=since,
347 + until=until,
348 + policy="reuse-same-day",
349 + current_config_checksum=source_config_checksum(sources),
350 + current_code_sha=None,
351 + )
352 +
353 + assert [article["title"] for article in reused] == ["Reused"]
354 + assert [source.name for source in to_crawl] == ["github-blog"]
355 + assert reused_statuses[0]["reused_same_day"] is True
356 + assert {decision["source"]: decision["decision"] for decision in decisions} == {
357 + "techcrunch": "reuse",
358 + "github-blog": "refresh",
359 + }
360 +
361 + def test_source_reuse_decisions_refreshes_malformed_article_artifact(self):
362 + sources = [NewsSourceConfig("techcrunch", "https://techcrunch.com/feed/")]
363 + since = datetime(2026, 5, 11, tzinfo=UTC)
364 + until = datetime(2026, 5, 18, tzinfo=UTC)
365 + payload = {
366 + "week": "2026-W21",
367 + "crawled_at": "2026-05-18T08:00:00Z",
368 + "crawl_window": {"since": iso_timestamp(since), "until": iso_timestamp(until)},
369 + "articles": [{"source": "techcrunch", "sources": 1, "title": "Malformed"}],
370 + "metadata": {
371 + "source_config_checksum": source_config_checksum(sources),
372 + "source_status": [{"source": "techcrunch", "success": True}],
373 + },
374 + }
375 +
376 + reused, to_crawl, reused_statuses, decisions = source_reuse_decisions(
377 + payload,
378 + sources,
379 + week="2026-W21",
380 + run_date=datetime(2026, 5, 18, tzinfo=UTC).date(),
381 + since=since,
382 + until=until,
383 + policy="reuse-same-day",
384 + current_config_checksum=source_config_checksum(sources),
385 + current_code_sha=None,
386 + )
387 +
388 + assert reused == []
389 + assert [source.name for source in to_crawl] == ["techcrunch"]
390 + assert reused_statuses == []
391 + assert decisions == [
392 + {"source": "techcrunch", "decision": "refresh", "reasons": ["artifact articles malformed"]}
393 + ]
394 +
395 + def test_source_reuse_decisions_refreshes_missing_code_fingerprint_when_required(self):
396 + sources = [NewsSourceConfig("techcrunch", "https://techcrunch.com/feed/")]
397 + since = datetime(2026, 5, 11, tzinfo=UTC)
398 + until = datetime(2026, 5, 18, tzinfo=UTC)
399 + payload = {
400 + "week": "2026-W21",
401 + "crawled_at": "2026-05-18T08:00:00Z",
402 + "crawl_window": {"since": iso_timestamp(since), "until": iso_timestamp(until)},
403 + "articles": [{"source": "techcrunch", "title": "Reused", "published_at": "2026-05-17T00:00:00Z"}],
404 + "metadata": {
405 + "source_config_checksum": source_config_checksum(sources),
406 + "source_status": [{"source": "techcrunch", "success": True}],
407 + },
408 + }
409 +
410 + reused, to_crawl, reused_statuses, decisions = source_reuse_decisions(
411 + payload,
412 + sources,
413 + week="2026-W21",
414 + run_date=datetime(2026, 5, 18, tzinfo=UTC).date(),
415 + since=since,
416 + until=until,
417 + policy="reuse-same-day",
418 + current_config_checksum=source_config_checksum(sources),
419 + current_code_sha="sha",
420 + )
421 +
422 + assert reused == []
423 + assert [source.name for source in to_crawl] == ["techcrunch"]
424 + assert reused_statuses == []
425 + assert decisions == [
426 + {"source": "techcrunch", "decision": "refresh", "reasons": ["crawler/config fingerprint mismatch"]}
427 + ]
428 +
429 @pytest.mark.parametrize(
430 "feed_url",
431 [
@@ -636,6 +748,28 @@ class TestSameDaySourceReuse:
748 assert [source.name for source in pending] == ["alpha", "beta"]
749 assert {item["action"] for item in summary} == {"stale"}
750
751 + def test_rejects_missing_code_fingerprint_when_required(self, tmp_path):
752 + from scripts.techcrunch_crawler import plan_source_reuse, source_config_checksum
753 + sources = self._sources()
754 + path = tmp_path / "external.json"
755 + now = datetime(2026, 5, 19, 9, 0, tzinfo=UTC)
756 + self._write_previous(path, crawled_at=now)
757 +
758 + reused, pending, summary, _, _ = plan_source_reuse(
759 + path,
760 + sources,
761 + now=now,
762 + since=datetime(2026, 5, 12, tzinfo=UTC),
763 + until=datetime(2026, 5, 19, tzinfo=UTC),
764 + config_checksum=source_config_checksum(sources),
765 + current_code_sha="sha",
766 + )
767 +
768 + assert reused == []
769 + assert [source.name for source in pending] == ["alpha", "beta"]
770 + assert {item["action"] for item in summary} == {"stale"}
771 + assert all("crawler/config fingerprint mismatch" in item["reasons"] for item in summary)
772 +
773 def test_partial_rerun_reuses_success_and_fetches_failed(self, tmp_path):
774 from scripts.techcrunch_crawler import plan_source_reuse, source_config_checksum
775 sources = self._sources()