fix(publish): enforce source-bound raw restore (#574)
* feat(publish): add immutable raw store with hash-verified restore (#570) Implement durable raw evidence storage on the existing publish branch under immutable week/source_run_id paths. A repeated write to the same path fails instead of replacing bytes, preserving provenance. Key changes: - publish_safety.py: store-raw and restore-raw commands with immutable directory creation, SHA-256 hash verification, and manifest.json with source run/artifact identity. - publish_manifest.py: restore verification integrates with eligibility — run_mode=restore requires source_run_id + raw_store_manifest, verifies all hashes before accepting restored inputs, and records provenance in the publish manifest. - rerun_modes.py: source_run_id required for restore mode, rejected for other modes, must be numeric. - crawl-and-publish.yml: stages data/raw-store/ before cached-diff eval, stores raw evidence during publish commit, restores from immutable store during rebuild, passes source_run_id through manifest creation. - sync-publish-to-main.yml: git add -A before cached diff evaluation so restored/generated files are staged first. - Documentation: artifact role (90-day transport) vs durable publish-branch role, restore requirements, operator examples. - Tests: store immutability, overwrite refusal, hash-verified restore, restore requires source_run_id, hash mismatch rejection, staging order. #569 W23-W29 evidence and provenance contract preserved untouched. Closes #570 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix(publish): enforce source-bound raw restore (#570) Closes #570 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix(publish): validate raw-store artifact name generically and derive WEEK in UTC Address Copilot review on jmservera/SquadScope#574: - validate_raw_store_manifest no longer hard-codes source_artifact.name == 'raw-data'; it now requires a present, non-empty name so manifests written with a custom --source-artifact-name remain validatable/restorable. - crawl-and-publish commit step derives WEEK with 'date -u' for consistency with the other UTC ISO-week derivations, avoiding week drift on non-UTC (e.g. self-hosted) runners. - tests: add restore coverage for a custom --source-artifact-name. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: jmservera <jmservera@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
Jul 17, 2026 at 18:04 UTC
651793261051a956ec8f8a5800f79f6ad1637611
13 files changed
+839
-20
.github/workflows/crawl-and-publish.yml
+62
-12
@@ -35,7 +35,12 @@ on:
35
default: false
36
type: boolean
37
rebuild_week:
38
- description: 'Restore a specific past week (e.g. 2026-W21). Requires run_mode=restore; skips crawl and hydrates from publish.'
38
+ description: 'Restore a specific past week (e.g. 2026-W21). Requires run_mode=restore and source_run_id; skips crawl and hydrates immutable raw evidence from publish.'
39
+ required: false
40
+ default: ''
41
+ type: string
42
+ source_run_id:
43
+ description: 'Required for restore: source workflow run ID selecting data/raw-store/<week>/<source_run_id>/ on publish.'
44
required: false
45
default: ''
46
type: string
@@ -78,10 +83,11 @@ jobs:
83
RUN_MODE: ${{ inputs.run_mode || 'normal' }}
84
SOURCE_REFRESH_POLICY: ${{ inputs.source_refresh_policy || 'reuse-same-day' }}
85
REBUILD_WEEK: ${{ inputs.rebuild_week || '' }}
86
+ SOURCE_RUN_ID: ${{ inputs.source_run_id || '' }}
87
PUBLISH_RELEASE: ${{ inputs.publish_release || false }}
88
run: |
89
set -euo pipefail
84
- ARGS=(--run-mode "$RUN_MODE" --source-refresh-policy "$SOURCE_REFRESH_POLICY" --rebuild-week "$REBUILD_WEEK" --summary-json data/diagnostics/rerun-mode.json)
90
+ ARGS=(--run-mode "$RUN_MODE" --source-refresh-policy "$SOURCE_REFRESH_POLICY" --rebuild-week "$REBUILD_WEEK" --source-run-id "$SOURCE_RUN_ID" --summary-json data/diagnostics/rerun-mode.json)
91
if [ "$PUBLISH_RELEASE" = "true" ]; then
92
ARGS+=(--publish-release)
93
fi
@@ -163,6 +169,20 @@ jobs:
169
with:
170
python-version: '3.12'
171
172
+ - name: Restore immutable raw evidence
173
+ if: ${{ inputs.rebuild_week }}
174
+ env:
175
+ REBUILD_WEEK: ${{ inputs.rebuild_week }}
176
+ SOURCE_RUN_ID: ${{ inputs.source_run_id }}
177
+ run: |
178
+ set -euo pipefail
179
+ git fetch origin publish
180
+ RAW_STORE_DIR="data/raw-store/${REBUILD_WEEK}/${SOURCE_RUN_ID}"
181
+ git checkout origin/publish -- "$RAW_STORE_DIR"
182
+ python3 scripts/publish_safety.py restore-raw \
183
+ --week "$REBUILD_WEEK" \
184
+ --source-run-id "$SOURCE_RUN_ID"
185
+
186
- name: Run crawler
187
if: ${{ !inputs.rebuild_week }}
188
env:
@@ -220,12 +240,14 @@ jobs:
240
PY
241
242
- name: Upload raw crawl artifact
243
+ id: raw-artifact
244
if: always()
245
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
246
with:
247
name: raw-data
248
path: data/raw/
249
if-no-files-found: warn
250
+ retention-days: 90
251
252
- name: Upload snapshot artifact
253
if: always()
@@ -249,10 +271,13 @@ jobs:
271
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
272
DATA_BRANCH: publish
273
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
274
+ RAW_ARTIFACT_ID: ${{ steps.raw-artifact.outputs.artifact-id }}
275
+ SOURCE_HEAD_SHA: ${{ github.sha }}
276
run: |
277
set -euo pipefail
278
git config user.name "github-actions[bot]"
279
git config user.email "github-actions[bot]@users.noreply.github.com"
280
+ cp scripts/publish_safety.py publish-safety-tool.py
281
# Save crawl output before switching branches
282
cp -r data/raw crawl-raw-backup
283
cp -r data/snapshots crawl-snapshots-backup
@@ -269,15 +294,31 @@ jobs:
294
cp -r crawl-raw-backup/* data/raw/ 2>/dev/null || true
295
cp -r crawl-snapshots-backup/* data/snapshots/ 2>/dev/null || true
296
rm -rf crawl-raw-backup crawl-snapshots-backup
272
- if git diff --quiet -- data/raw data/snapshots; then
273
- echo "No changes to data/raw or data/snapshots. Skipping."
297
+ WEEK=$(date -u +%Y-W%V)
298
+ RAW_PATH_ARGS=(--path "data/raw/${WEEK}.json")
299
+ for optional_raw in \
300
+ "data/raw/${WEEK}-external-news.json" \
301
+ "data/raw/${WEEK}-techcrunch.json"
302
+ do
303
+ [ -f "$optional_raw" ] && RAW_PATH_ARGS+=(--path "$optional_raw")
304
+ done
305
+ python3 publish-safety-tool.py store-raw \
306
+ --week "$WEEK" \
307
+ --source-run-id "$GITHUB_RUN_ID" \
308
+ --source-artifact-id "$RAW_ARTIFACT_ID" \
309
+ --source-artifact-name raw-data \
310
+ --source-head-sha "$SOURCE_HEAD_SHA" \
311
+ "${RAW_PATH_ARGS[@]}"
312
+ rm -f publish-safety-tool.py
313
+ git add data/raw/ data/snapshots/ data/raw-store/
314
+ if git diff --cached --quiet; then
315
+ echo "No raw or snapshot changes to commit after staging."
316
exit 0
317
fi
318
COUNTER=$(cat .squad/run-counter.txt 2>/dev/null || echo 0)
319
COUNTER=$((COUNTER + 1))
320
printf '%s\n' "$COUNTER" > .squad/run-counter.txt
279
- WEEK=$(date +%Y-W%V)
280
- git add data/raw/ data/snapshots/ .squad/run-counter.txt
321
+ git add .squad/run-counter.txt
322
git diff --cached --quiet && exit 0
323
git commit -m "data: weekly crawl $WEEK [run #${GITHUB_RUN_ID}]"
324
if [ -n "$EXPECTED_PUBLISH_SHA" ]; then
@@ -313,7 +354,6 @@ jobs:
354
persist-credentials: false
355
356
- name: Download raw crawl artifact
316
- if: ${{ !inputs.rebuild_week }}
357
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
358
with:
359
name: raw-data
@@ -323,6 +363,7 @@ jobs:
363
if: ${{ inputs.rebuild_week }}
364
env:
365
REBUILD_WEEK: ${{ inputs.rebuild_week }}
366
+ SOURCE_RUN_ID: ${{ inputs.source_run_id }}
367
run: |
368
set -euo pipefail
369
if ! [[ "$REBUILD_WEEK" =~ ^[0-9]{4}-W[0-9]{2}$ ]]; then
@@ -330,10 +371,11 @@ jobs:
371
exit 1
372
fi
373
git fetch origin publish
333
- mkdir -p data/raw data/analyzed data/snapshots
334
- git checkout origin/publish -- "data/raw/${REBUILD_WEEK}.json"
335
- git checkout origin/publish -- "data/raw/${REBUILD_WEEK}-external-news.json" 2>/dev/null || true
336
- git checkout origin/publish -- "data/raw/${REBUILD_WEEK}-techcrunch.json" 2>/dev/null || true
374
+ mkdir -p data/analyzed data/snapshots
375
+ git checkout origin/publish -- "data/raw-store/${REBUILD_WEEK}/${SOURCE_RUN_ID}"
376
+ python3 scripts/publish_safety.py restore-raw \
377
+ --week "$REBUILD_WEEK" \
378
+ --source-run-id "$SOURCE_RUN_ID"
379
git checkout origin/publish -- "data/snapshots/${REBUILD_WEEK}-stars.json" 2>/dev/null || true
380
# Also hydrate ALL prior analyzed files so rollups have correct historical context
381
git ls-tree -r --name-only origin/publish -- data/analyzed/ | while read -r f; do
@@ -865,12 +907,20 @@ jobs:
907
MANIFEST_FILE: ${{ steps.analysis-context.outputs.publish_manifest_file }}
908
RUN_MODE: ${{ steps.analysis-context.outputs.run_mode }}
909
SOURCE_REFRESH_POLICY: ${{ steps.analysis-context.outputs.source_refresh_policy }}
910
+ SOURCE_RUN_ID: ${{ inputs.source_run_id || '' }}
911
GATE_REPORT: ${{ steps.analysis-context.outputs.analysis_gate_report_file }}
912
PREFLIGHT_REPORT: ${{ steps.prompt-preflight.outputs.preflight_report_json }}
913
run: |
914
set -euo pipefail
915
git fetch origin publish 2>/dev/null && git checkout origin/publish -- "$PUBLISHED_SUMMARY" 2>/dev/null || true
916
ARTIFACT_ARGS=()
917
+ RESTORE_ARGS=()
918
+ if [ "$RUN_MODE" = "restore" ]; then
919
+ RESTORE_ARGS=(
920
+ --source-run-id "$SOURCE_RUN_ID"
921
+ --raw-store-manifest "data/raw-store/${WEEK}/${SOURCE_RUN_ID}/manifest.json"
922
+ )
923
+ fi
924
for candidate in \
925
"external_news=data/raw/${WEEK}-external-news.json" \
926
"techcrunch_news=data/raw/${WEEK}-techcrunch.json" \
@@ -897,6 +947,7 @@ jobs:
947
--source-refresh-policy "$SOURCE_REFRESH_POLICY" \
948
--gate-report "$GATE_REPORT" \
949
--output "$MANIFEST_FILE" \
950
+ "${RESTORE_ARGS[@]}" \
951
"${ARTIFACT_ARGS[@]}"
952
953
- name: Assert candidate is eligible for promotion
@@ -1364,4 +1415,3 @@ jobs:
1415
--label squad \
1416
--body "$BODY"
1417
fi
1367
-
.github/workflows/sync-publish-to-main.yml
+3
-3
@@ -61,13 +61,13 @@ jobs:
61
# pages cannot drift from the weekly content in the generated sync PR.
62
python3 scripts/generate_rollups.py
63
64
- # Check if there are any changes
65
- if git diff --cached --quiet && git diff --quiet; then
64
+ # Stage restored/generated files before evaluating the cached diff.
65
+ git add -A
66
+ if git diff --cached --quiet; then
67
echo "No changes to sync."
68
exit 0
69
fi
70
70
- git add -A
71
if git diff --cached --name-only | grep -E '^\.squad/' >/dev/null; then
72
echo "::error::Refusing to sync .squad state from publish to main."
73
git diff --cached --name-only | grep -E '^\.squad/' || true
docs/matrix-crawl-runbook.md
+6
-3
@@ -42,7 +42,8 @@ gh workflow run crawl-and-publish.yml \
42
# Restore a specific past week
43
gh workflow run crawl-and-publish.yml \
44
-f run_mode=restore \
45
- -f rebuild_week=2026-W23
45
+ -f rebuild_week=2026-W23 \
46
+ -f source_run_id=26753498571
47
```
48
49
### Run modes
@@ -71,7 +72,8 @@ gh workflow run crawl-and-publish.yml \
72
| Artifact | Location | Purpose |
73
|----------|----------|---------|
74
| `crawl-cache` | Actions artifact | GitHub API response cache |
74
-| `raw-data` | Actions artifact | Canonical crawl payloads |
75
+| `raw-data` | Actions artifact (90 days) | Job transport, same-day reuse, and emergency recovery; not durable storage |
76
+| Immutable raw store | `publish:data/raw-store/<week>/<source_run_id>/` | Durable source-bound payloads, hashes, and artifact provenance |
77
| `crawl-snapshots` | Actions artifact | Star/trending snapshots |
78
| Rerun mode summary | `data/diagnostics/rerun-mode.json` | Mode validation output |
79
| External news | `data/raw/{week}-external-news.json` | RSS crawl result |
@@ -181,7 +183,8 @@ gh workflow run crawl-and-publish.yml \
183
# 3. If a specific week needs rebuilding
184
gh workflow run crawl-and-publish.yml \
185
-f run_mode=restore \
184
- -f rebuild_week=2026-W23
186
+ -f rebuild_week=2026-W23 \
187
+ -f source_run_id=26753498571
188
189
# 4. If analysis failed but crawl succeeded, re-run from artifacts
190
gh run rerun <run-id> --failed
docs/operator-guide.md
+38
-1
@@ -195,7 +195,7 @@ All rerun modes are validated before any publishing side effects:
195
| `normal` (default) | ✓ Fresh | ✓ Guarded gates | Produce fresh analysis, publish if gates pass | Standard weekly run |
196
| `dry-run` | ✓ Fresh | ✗ Never | Build candidates only for inspection | Test analysis quality, verify gates, debug analysis |
197
| `candidate-only` | ✓ Fresh | ✗ Manifest blocks | Run crawl/analysis but hold for manual approval | Staged analysis, manual promotion workflow |
198
-| `restore` | ✗ Hydrate | ✓ Guarded gates | Regenerate prior week from published artifacts | Restore/audit trail, regenerate HTML/feeds |
198
+| `restore` | ✗ Hydrate | ✓ Guarded gates | Regenerate a prior week from one source-bound immutable raw run | Restore/audit trail, regenerate HTML/feeds |
199
| `force-replace` | ✓ Fresh | ✓ Guarded gates | Explicit replacement run, gates still enforce | Planned content refresh, operator override |
200
201
##### Source refresh policies
@@ -215,9 +215,46 @@ Same-day artifact reuse is safe by design:
215
216
Invalid combinations fail immediately with clear error messages:
217
- `rebuild_week` without `run_mode=restore`
218
+- `run_mode=restore` without both `rebuild_week` and `source_run_id`
219
- `run_mode=restore` with `source_refresh_policy=force-refresh`
220
- `publish_release=true` with `dry-run` or `candidate-only`
221
222
+##### Durable raw evidence and restore
223
+
224
+Each publishing crawl writes the current week's raw payloads to the existing
225
+`publish` branch under:
226
+
227
+```text
228
+data/raw-store/<week>/<source_run_id>/
229
+```
230
+
231
+The run directory is immutable: a repeated write to the same week/run path fails
232
+instead of replacing files. Its `manifest.json` records the source workflow run,
233
+`raw-data` artifact ID/name, source head SHA, original paths, sizes, and SHA-256
234
+hashes.
235
+
236
+Restore must identify that exact source run:
237
+
238
+```bash
239
+gh workflow run crawl-and-publish.yml \
240
+ -R YOUR_USERNAME/SquadScope \
241
+ -f run_mode=restore \
242
+ -f rebuild_week=2026-W23 \
243
+ -f source_run_id=26753498571
244
+```
245
+
246
+The workflow verifies the immutable manifest identity and every stored hash before
247
+copying any file into `data/raw/`. The selected manifest is authoritative for the
248
+week, so same-week raw files left by the checkout or artifact overlay but absent
249
+from that source run are removed before analysis. The publish eligibility manifest
250
+records the verified source run/artifact provenance and rechecks restored input
251
+hashes before promotion.
252
+
253
+The GitHub Actions `raw-data` artifact has **90-day retention** and remains a
254
+transport mechanism for jobs, same-day reuse, and emergency recovery only. It is not
255
+the durable raw store. The durable copy is the immutable week/run directory on
256
+`publish`; it is intentionally not synced into `main`.
257
+
258
### Option C: Run individual stages locally
259
260
For debugging or testing, run stages separately:
docs/pipeline-validation.md
+6
@@ -33,6 +33,7 @@ Required secrets/tokens:
33
- `data/raw/YYYY-WNN-external-news.json`
34
- `data/snapshots/YYYY-WNN-stars.json`
35
- `raw-data` artifact
36
+- Immutable raw evidence at `publish:data/raw-store/YYYY-WNN/<source_run_id>/`
37
- `crawl-snapshots` artifact
38
- `crawl-cache` artifact
39
- Commit to `main` for `data/raw/` and `data/snapshots/`
@@ -44,6 +45,9 @@ Required secrets/tokens:
45
- Optional per-source RSS failures are warnings with a valid partial artifact; malformed config/schema/checksum errors fail the crawl step
46
- Snapshot file is written for the same ISO week
47
- Cache artifact uploads even on partial failures
48
+- `raw-data` has explicit 90-day retention for transport/emergency recovery only
49
+- The publish-branch raw store refuses an existing week/run destination and records
50
+ source run/artifact identity plus per-file SHA-256 hashes
51
- Job permissions include `actions: read` and `contents: write` at workflow level for cache restore and commits
52
53
### 2. Analyze
@@ -65,6 +69,8 @@ Required secrets/tokens:
69
70
**Success criteria**
71
- Current raw file week matches the run week
72
+- Restore mode requires `source_run_id`, hydrates the matching immutable raw-store
73
+ directory, and verifies all hashes before analysis accepts the inputs
74
- Correlation and press-context steps consume compact external-news data with legacy `YYYY-WNN-techcrunch.json` fallback
75
- Press context preserves source names, article URLs/titles/dates, strong-vs-weak labels, and partial-source caveats while staying under the ~8k token budget
76
- Analysis preflight writes `analysis-preflight.json` with deterministic prompt component byte/token/checksum metadata and raw/prompt evidence inventories before Copilot is invoked.
scripts/publish_manifest.py
+107
@@ -9,6 +9,11 @@ from datetime import UTC, datetime
9
from pathlib import Path
10
from typing import Any
11
12
+try:
13
+ from scripts import publish_safety
14
+except ModuleNotFoundError: # pragma: no cover - direct script execution path
15
+ import publish_safety # type: ignore[no-redef]
16
+
17
SCHEMA_VERSION = "publish_eligibility_v1"
18
AI_SOURCES = {"copilot-cli"}
19
RUN_MODES = {"normal", "dry-run", "restore", "force-replace", "candidate-only"}
@@ -38,6 +43,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
43
create = subparsers.add_parser("create", help="Create a publish eligibility manifest.")
44
create.add_argument("--week", required=True)
45
create.add_argument("--run-id", required=True)
46
+ create.add_argument("--root", default=".", type=Path)
47
create.add_argument("--current-datetime", required=True)
48
create.add_argument("--summary", required=True, type=Path)
49
create.add_argument(
@@ -56,6 +62,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
62
)
63
create.add_argument("--validation-status", choices=["passed", "failed"], required=True)
64
create.add_argument("--run-mode", choices=sorted(RUN_MODES), default="normal")
65
+ create.add_argument("--source-run-id", default="")
66
+ create.add_argument(
67
+ "--raw-store-manifest",
68
+ type=Path,
69
+ help="Immutable raw store manifest required for run_mode=restore.",
70
+ )
71
create.add_argument(
72
"--source-refresh-policy", choices=sorted(SOURCE_REFRESH_POLICIES), default="reuse-same-day"
73
)
@@ -139,6 +151,77 @@ def load_json(path: Path) -> dict[str, Any] | None:
151
return payload if isinstance(payload, dict) else None
152
153
154
+def restore_verification(args: argparse.Namespace) -> tuple[dict[str, Any], list[str]]:
155
+ required = args.run_mode == "restore"
156
+ source_run_id = args.source_run_id.strip()
157
+ manifest_path = args.raw_store_manifest
158
+ resolved_manifest_path = (
159
+ manifest_path
160
+ if manifest_path is None or manifest_path.is_absolute()
161
+ else args.root / manifest_path
162
+ )
163
+ result: dict[str, Any] = {
164
+ "required": required,
165
+ "verified": False,
166
+ "source_run_id": source_run_id or None,
167
+ "manifest_path": manifest_path.as_posix() if manifest_path else None,
168
+ "manifest_sha256": (
169
+ sha256_file(resolved_manifest_path) if resolved_manifest_path else None
170
+ ),
171
+ "source_artifact": None,
172
+ "files": [],
173
+ }
174
+ reasons: list[str] = []
175
+ if not required:
176
+ if source_run_id or manifest_path is not None:
177
+ reasons.append("restore provenance is only allowed with run_mode=restore")
178
+ return result, reasons
179
+ if not source_run_id:
180
+ reasons.append("run_mode=restore requires source_run_id")
181
+ if manifest_path is None:
182
+ reasons.append("run_mode=restore requires raw_store_manifest")
183
+ if reasons:
184
+ return result, reasons
185
+
186
+ try:
187
+ payload, verified = publish_safety.validate_raw_store_manifest(
188
+ args.root,
189
+ manifest_path,
190
+ expected_week=args.week,
191
+ expected_source_run_id=source_run_id,
192
+ )
193
+ except SystemExit as exc:
194
+ return result, [str(exc)]
195
+
196
+ verified_files: list[dict[str, Any]] = []
197
+ for _, restored_path, entry in verified:
198
+ if not restored_path.exists() or not restored_path.is_file():
199
+ reasons.append(f"Restored raw input is missing: {entry['original_path']}")
200
+ continue
201
+ if restored_path.stat().st_size != entry.get("size_bytes"):
202
+ reasons.append(f"Restored raw input size mismatch: {entry['original_path']}")
203
+ continue
204
+ if sha256_file(restored_path) != entry.get("sha256"):
205
+ reasons.append(f"Restored raw input checksum mismatch: {entry['original_path']}")
206
+ continue
207
+ verified_files.append(
208
+ {
209
+ "original_path": entry["original_path"],
210
+ "size_bytes": entry["size_bytes"],
211
+ "sha256": entry["sha256"],
212
+ }
213
+ )
214
+
215
+ result.update(
216
+ {
217
+ "verified": not reasons and len(verified_files) == len(verified),
218
+ "source_artifact": payload.get("source_artifact"),
219
+ "files": verified_files,
220
+ }
221
+ )
222
+ return result, reasons
223
+
224
+
225
def load_preflight(
226
path: Path | None, *, required: bool = False
227
) -> tuple[dict[str, Any] | None, list[str]]:
@@ -564,6 +647,7 @@ def create_manifest(args: argparse.Namespace) -> int:
647
model_status = publishable_model_status(args.analysis_model)
648
preflight, preflight_reasons = load_preflight(args.preflight_report, required=ai_status == "ai")
649
gate_report = load_gate_report(args.gate_report)
650
+ restore, restore_reasons = restore_verification(args)
651
candidate_metadata = markdown_metadata(args.summary)
652
published_status = published_summary_status(args.published_summary, args.week)
653
candidate_exists = args.summary.exists()
@@ -649,6 +733,7 @@ def create_manifest(args: argparse.Namespace) -> int:
733
if not gates_passed:
734
reasons.extend(gate_reasons(gate_report))
735
reasons.extend(artifact_reasons)
736
+ reasons.extend(restore_reasons)
737
reasons.extend(comparison_reasons)
738
739
eligible = (
@@ -657,6 +742,7 @@ def create_manifest(args: argparse.Namespace) -> int:
742
and validation_passed
743
and gates_passed
744
and not artifact_reasons
745
+ and not restore_reasons
746
and not comparison_reasons
747
and not preflight_reasons
748
and mode_allows_promotion
@@ -680,6 +766,7 @@ def create_manifest(args: argparse.Namespace) -> int:
766
"run_mode": args.run_mode,
767
"source_refresh_policy": args.source_refresh_policy,
768
"run_started_at": args.current_datetime,
769
+ "restore": restore,
770
"candidate_summary_path": args.summary.as_posix(),
771
"candidate_content_path": candidate_content.as_posix(),
772
"promotion_eligible": eligible,
@@ -817,6 +904,26 @@ def assert_eligible(args: argparse.Namespace) -> int:
904
raise SystemExit(PROMOTION_MANIFEST_ROOT_ERROR)
905
if payload.get("schema_version") != SCHEMA_VERSION:
906
raise SystemExit(f"Unsupported publish manifest schema: {payload.get('schema_version')!r}")
907
+ if payload.get("run_mode") == "restore":
908
+ restore = payload.get("restore")
909
+ if (
910
+ not isinstance(restore, dict)
911
+ or restore.get("required") is not True
912
+ or restore.get("verified") is not True
913
+ or not restore.get("source_run_id")
914
+ or not restore.get("manifest_sha256")
915
+ or not isinstance(restore.get("source_artifact"), dict)
916
+ ):
917
+ raise SystemExit("Manifest lacks verified source-bound raw restore provenance.")
918
+ restored_files = restore.get("files")
919
+ if not isinstance(restored_files, list) or not restored_files:
920
+ raise SystemExit("Manifest lacks verified restored raw files.")
921
+ for entry in restored_files:
922
+ if not isinstance(entry, dict) or not isinstance(entry.get("original_path"), str):
923
+ raise SystemExit("Manifest restored raw file entry is malformed.")
924
+ _, path = publish_safety.require_raw_path(Path.cwd(), entry["original_path"])
925
+ if sha256_file(path) != entry.get("sha256"):
926
+ raise SystemExit(f"Restored raw input checksum mismatch: {path}")
927
analysis = payload.get("analysis")
928
if not isinstance(analysis, dict):
929
raise SystemExit("Manifest lacks publishable AI provenance.")
scripts/publish_safety.py
+258
@@ -4,6 +4,7 @@ from __future__ import annotations
4
import argparse
5
import hashlib
6
import json
7
+import re
8
import shutil
9
from datetime import UTC, datetime
10
from pathlib import Path
@@ -11,6 +12,9 @@ from typing import Any
12
13
BACKUP_SCHEMA_VERSION = "publish_backup_v1"
14
PUBLISH_MANIFEST_SCHEMA_VERSION = "publish_eligibility_v1"
15
+RAW_STORE_SCHEMA_VERSION = "raw_store_v1"
16
+WEEK_PATTERN = re.compile(r"^[0-9]{4}-W[0-9]{2}$")
17
+SAFE_COMPONENT_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$")
18
19
20
def sha256_file(path: Path) -> str | None:
@@ -53,6 +57,33 @@ def path_under_root(root: Path, value: Path) -> tuple[Path, Path]:
57
return relative, resolved_path
58
59
60
+def require_safe_component(value: str, *, label: str) -> str:
61
+ candidate = value.strip()
62
+ if not candidate or not SAFE_COMPONENT_PATTERN.fullmatch(candidate):
63
+ raise SystemExit(f"Invalid {label}: {value!r}")
64
+ return candidate
65
+
66
+
67
+def require_week(value: str) -> str:
68
+ week = value.strip()
69
+ if not WEEK_PATTERN.fullmatch(week):
70
+ raise SystemExit(f"Invalid week format. Expected YYYY-WNN, got: {value!r}")
71
+ return week
72
+
73
+
74
+def require_raw_path(root: Path, value: str) -> tuple[Path, Path]:
75
+ relative = relpath_under_root(root, value)
76
+ if relative.parts[:2] != ("data", "raw"):
77
+ raise SystemExit(f"Raw store paths must live under data/raw/: {value}")
78
+ return relative, root / relative
79
+
80
+
81
+def is_week_raw_json(path: Path, week: str) -> bool:
82
+ return path.suffix == ".json" and (
83
+ path.name == f"{week}.json" or path.name.startswith(f"{week}-")
84
+ )
85
+
86
+
87
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
88
parser = argparse.ArgumentParser(description="Publish-branch backup and restore safeguards.")
89
subparsers = parser.add_subparsers(dest="command", required=True)
@@ -83,6 +114,31 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
114
restore.add_argument("--root", default=".", type=Path)
115
restore.add_argument("--backup-manifest", required=True, type=Path)
116
117
+ store_raw_parser = subparsers.add_parser(
118
+ "store-raw", help="Store raw evidence under an immutable week/source-run path."
119
+ )
120
+ store_raw_parser.add_argument("--root", default=".", type=Path)
121
+ store_raw_parser.add_argument("--week", required=True)
122
+ store_raw_parser.add_argument("--source-run-id", required=True)
123
+ store_raw_parser.add_argument("--source-artifact-id", required=True)
124
+ store_raw_parser.add_argument("--source-artifact-name", default="raw-data")
125
+ store_raw_parser.add_argument("--source-head-sha", required=True)
126
+ store_raw_parser.add_argument("--store-root", default=Path("data/raw-store"), type=Path)
127
+ store_raw_parser.add_argument(
128
+ "--path",
129
+ action="append",
130
+ required=True,
131
+ help="Week-scoped data/raw file to preserve.",
132
+ )
133
+
134
+ restore_raw_parser = subparsers.add_parser(
135
+ "restore-raw", help="Restore hash-verified raw evidence from a source workflow run."
136
+ )
137
+ restore_raw_parser.add_argument("--root", default=".", type=Path)
138
+ restore_raw_parser.add_argument("--week", required=True)
139
+ restore_raw_parser.add_argument("--source-run-id", required=True)
140
+ restore_raw_parser.add_argument("--store-root", default=Path("data/raw-store"), type=Path)
141
+
142
return parser.parse_args(argv)
143
144
@@ -203,12 +259,214 @@ def restore_backup(args: argparse.Namespace) -> int:
259
return 0
260
261
262
+def store_raw(args: argparse.Namespace) -> int:
263
+ root = args.root.resolve()
264
+ week = require_week(args.week)
265
+ source_run_id = require_safe_component(args.source_run_id, label="source_run_id")
266
+ source_artifact_id = args.source_artifact_id.strip()
267
+ source_artifact_name = args.source_artifact_name.strip()
268
+ source_head_sha = args.source_head_sha.strip()
269
+ if not source_artifact_id:
270
+ raise SystemExit("source_artifact_id is required.")
271
+ if not source_artifact_name:
272
+ raise SystemExit("source_artifact_name is required.")
273
+ if not source_head_sha:
274
+ raise SystemExit("source_head_sha is required.")
275
+
276
+ sources: list[tuple[Path, Path]] = []
277
+ seen_sources: set[Path] = set()
278
+ for raw_path in args.path:
279
+ relative, source = require_raw_path(root, raw_path)
280
+ if relative in seen_sources:
281
+ raise SystemExit(f"Duplicate raw store source: {relative.as_posix()}")
282
+ seen_sources.add(relative)
283
+ if not source.exists() or not source.is_file():
284
+ raise SystemExit(f"Raw store source must be a regular file: {relative.as_posix()}")
285
+ if relative.name != f"{week}.json" and not relative.name.startswith(f"{week}-"):
286
+ raise SystemExit(f"Raw store source does not belong to {week}: {relative.as_posix()}")
287
+ sources.append((relative, source))
288
+
289
+ store_root = root / relpath_under_root(root, args.store_root.as_posix())
290
+ destination = store_root / week / source_run_id
291
+ if destination.exists():
292
+ raise SystemExit(f"Refusing to overwrite immutable raw store: {destination}")
293
+ destination.mkdir(parents=True, exist_ok=False)
294
+
295
+ entries: list[dict[str, Any]] = []
296
+ try:
297
+ for relative, source in sorted(sources):
298
+ stored = destination / "files" / relative
299
+ stored.parent.mkdir(parents=True, exist_ok=True)
300
+ shutil.copy2(source, stored)
301
+ source_hash = sha256_file(source)
302
+ stored_hash = sha256_file(stored)
303
+ if source_hash is None or stored_hash != source_hash:
304
+ raise SystemExit(f"Raw store checksum mismatch while copying {relative.as_posix()}")
305
+ entries.append(
306
+ {
307
+ "week": week,
308
+ "artifact_id": source_artifact_id,
309
+ "source_run_id": source_run_id,
310
+ "head_sha": source_head_sha,
311
+ "original_path": relative.as_posix(),
312
+ "stored_path": stored.relative_to(root).as_posix(),
313
+ "size_bytes": stored.stat().st_size,
314
+ "sha256": stored_hash,
315
+ }
316
+ )
317
+
318
+ manifest = {
319
+ "schema_version": RAW_STORE_SCHEMA_VERSION,
320
+ "created_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
321
+ "week": week,
322
+ "source_run_id": source_run_id,
323
+ "source_artifact": {
324
+ "id": source_artifact_id,
325
+ "name": source_artifact_name,
326
+ "head_sha": source_head_sha,
327
+ "retention_days": 90,
328
+ },
329
+ "files": entries,
330
+ }
331
+ manifest_path = destination / "manifest.json"
332
+ manifest_path.write_text(
333
+ json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
334
+ )
335
+ except BaseException:
336
+ shutil.rmtree(destination, ignore_errors=True)
337
+ raise
338
+
339
+ print(f"Created immutable raw store: {destination.relative_to(root).as_posix()}")
340
+ return 0
341
+
342
+
343
+def validate_raw_store_manifest(
344
+ root: Path,
345
+ manifest_path: Path,
346
+ *,
347
+ expected_week: str,
348
+ expected_source_run_id: str,
349
+) -> tuple[dict[str, Any], list[tuple[Path, Path, dict[str, Any]]]]:
350
+ root = root.resolve()
351
+ week = require_week(expected_week)
352
+ source_run_id = require_safe_component(expected_source_run_id, label="source_run_id")
353
+ manifest_relative, manifest = path_under_root(root, manifest_path)
354
+ payload = load_json(manifest)
355
+ if payload is None:
356
+ raise SystemExit(f"Raw store manifest is missing or malformed: {manifest_relative}")
357
+ if payload.get("schema_version") != RAW_STORE_SCHEMA_VERSION:
358
+ raise SystemExit(f"Unsupported raw store schema: {payload.get('schema_version')!r}")
359
+ if payload.get("week") != week:
360
+ raise SystemExit(f"Raw store week mismatch: expected {week}, found {payload.get('week')!r}")
361
+ if payload.get("source_run_id") != source_run_id:
362
+ raise SystemExit(
363
+ "Raw store source_run_id mismatch: "
364
+ f"expected {source_run_id}, found {payload.get('source_run_id')!r}"
365
+ )
366
+ source_artifact = payload.get("source_artifact")
367
+ artifact_name = source_artifact.get("name") if isinstance(source_artifact, dict) else None
368
+ if (
369
+ not isinstance(source_artifact, dict)
370
+ or not source_artifact.get("id")
371
+ or not (isinstance(artifact_name, str) and artifact_name.strip())
372
+ or not source_artifact.get("head_sha")
373
+ ):
374
+ raise SystemExit("Raw store manifest lacks source artifact provenance.")
375
+ files = payload.get("files")
376
+ if not isinstance(files, list) or not files:
377
+ raise SystemExit("Raw store manifest has no files list.")
378
+
379
+ verified: list[tuple[Path, Path, dict[str, Any]]] = []
380
+ store_directory = manifest.parent.resolve()
381
+ store_directory_relative = manifest.parent.relative_to(root)
382
+ seen_original_paths: set[Path] = set()
383
+ for entry in files:
384
+ if not isinstance(entry, dict):
385
+ raise SystemExit("Raw store file entry is malformed.")
386
+ original_path = entry.get("original_path")
387
+ stored_path = entry.get("stored_path")
388
+ if not isinstance(original_path, str) or not isinstance(stored_path, str):
389
+ raise SystemExit("Raw store file entry lacks original_path or stored_path.")
390
+ if entry.get("week") != week or entry.get("source_run_id") != source_run_id:
391
+ raise SystemExit(f"Raw store file provenance mismatch for {original_path}")
392
+ if entry.get("artifact_id") != source_artifact.get("id") or entry.get(
393
+ "head_sha"
394
+ ) != source_artifact.get("head_sha"):
395
+ raise SystemExit(f"Raw store artifact provenance mismatch for {original_path}")
396
+
397
+ target_relative, target = require_raw_path(root, original_path)
398
+ if target_relative in seen_original_paths:
399
+ raise SystemExit(f"Duplicate raw store original_path: {original_path}")
400
+ seen_original_paths.add(target_relative)
401
+ stored_relative = relpath_under_root(root, stored_path)
402
+ expected_stored_relative = store_directory_relative / "files" / target_relative
403
+ if stored_relative != expected_stored_relative:
404
+ raise SystemExit(f"Raw store stored_path mismatch for {original_path}: {stored_path}")
405
+ stored = root / stored_relative
406
+ try:
407
+ stored.resolve().relative_to(store_directory)
408
+ except ValueError as exc:
409
+ raise SystemExit(
410
+ f"Raw store file must stay under its immutable run directory: {stored_path}"
411
+ ) from exc
412
+ if not stored.exists() or not stored.is_file():
413
+ raise SystemExit(f"Raw store file is missing: {stored_relative.as_posix()}")
414
+ if stored.stat().st_size != entry.get("size_bytes"):
415
+ raise SystemExit(f"Raw store size mismatch for {stored_relative.as_posix()}")
416
+ if sha256_file(stored) != entry.get("sha256"):
417
+ raise SystemExit(f"Raw store checksum mismatch for {stored_relative.as_posix()}")
418
+ verified.append((stored, target, entry))
419
+
420
+ required_raw_path = Path("data") / "raw" / f"{week}.json"
421
+ if required_raw_path not in seen_original_paths:
422
+ raise SystemExit(f"Raw store manifest lacks required payload: {required_raw_path}")
423
+ return payload, verified
424
+
425
+
426
+def restore_raw(args: argparse.Namespace) -> int:
427
+ root = args.root.resolve()
428
+ week = require_week(args.week)
429
+ source_run_id = require_safe_component(args.source_run_id, label="source_run_id")
430
+ store_root = root / relpath_under_root(root, args.store_root.as_posix())
431
+ manifest = store_root / week / source_run_id / "manifest.json"
432
+ _, verified = validate_raw_store_manifest(
433
+ root,
434
+ manifest,
435
+ expected_week=week,
436
+ expected_source_run_id=source_run_id,
437
+ )
438
+
439
+ expected_targets = {target for _, target, _ in verified}
440
+ raw_root = root / "data" / "raw"
441
+ if raw_root.exists():
442
+ for existing in sorted(raw_root.rglob("*.json")):
443
+ if (
444
+ existing.is_file()
445
+ and is_week_raw_json(existing, week)
446
+ and existing not in expected_targets
447
+ ):
448
+ existing.unlink()
449
+
450
+ for stored, target, _ in verified:
451
+ target.parent.mkdir(parents=True, exist_ok=True)
452
+ shutil.copy2(stored, target)
453
+ print(
454
+ "Restored hash-verified raw evidence: "
455
+ f"week={week} source_run_id={source_run_id} files={len(verified)}"
456
+ )
457
+ return 0
458
+
459
+
460
def main(argv: list[str] | None = None) -> int:
461
args = parse_args(argv)
462
if args.command == "backup-existing":
463
return backup_existing(args)
464
if args.command == "restore-backup":
465
return restore_backup(args)
466
+ if args.command == "store-raw":
467
+ return store_raw(args)
468
+ if args.command == "restore-raw":
469
+ return restore_raw(args)
470
raise AssertionError(args.command)
471
472
scripts/rerun_modes.py
+15
@@ -5,11 +5,14 @@ from __future__ import annotations
5
6
import argparse
7
import json
8
+import re
9
from dataclasses import dataclass
10
from pathlib import Path
11
12
RUN_MODES = {"normal", "dry-run", "restore", "force-replace", "candidate-only"}
13
SOURCE_REFRESH_POLICIES = {"reuse-same-day", "refresh-missing-stale", "force-refresh"}
14
+WEEK_PATTERN = re.compile(r"^[0-9]{4}-W[0-9]{2}$")
15
+RUN_ID_PATTERN = re.compile(r"^[0-9]+$")
16
17
18
@dataclass(frozen=True)
@@ -27,6 +30,7 @@ def validate_modes(
30
run_mode: str,
31
source_refresh_policy: str,
32
rebuild_week: str = "",
33
+ source_run_id: str = "",
34
publish_release: bool = False,
35
) -> ModeDecision:
36
reasons: list[str] = []
@@ -36,8 +40,16 @@ def validate_modes(
40
reasons.append(f"invalid source_refresh_policy: {source_refresh_policy}")
41
if rebuild_week and run_mode != "restore":
42
reasons.append("rebuild_week is a restore operation and requires run_mode=restore")
43
+ if rebuild_week and not WEEK_PATTERN.fullmatch(rebuild_week):
44
+ reasons.append("rebuild_week must use YYYY-WNN format")
45
if run_mode == "restore" and not rebuild_week:
46
reasons.append("run_mode=restore requires rebuild_week=YYYY-WNN")
47
+ if run_mode == "restore" and not source_run_id:
48
+ reasons.append("run_mode=restore requires source_run_id")
49
+ if source_run_id and run_mode != "restore":
50
+ reasons.append("source_run_id is only allowed with run_mode=restore")
51
+ if source_run_id and not RUN_ID_PATTERN.fullmatch(source_run_id):
52
+ reasons.append("source_run_id must be a numeric GitHub Actions workflow run ID")
53
if run_mode in {"dry-run", "candidate-only"} and publish_release:
54
reasons.append(f"publish_release is not allowed with run_mode={run_mode}")
55
if run_mode == "restore" and source_refresh_policy == "force-refresh":
@@ -68,6 +80,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
80
parser.add_argument("--run-mode", default="normal")
81
parser.add_argument("--source-refresh-policy", default="reuse-same-day")
82
parser.add_argument("--rebuild-week", default="")
83
+ parser.add_argument("--source-run-id", default="")
84
parser.add_argument("--publish-release", action="store_true")
85
parser.add_argument("--summary-json", type=Path)
86
return parser.parse_args(argv)
@@ -79,6 +92,7 @@ def main(argv: list[str] | None = None) -> int:
92
run_mode=args.run_mode,
93
source_refresh_policy=args.source_refresh_policy,
94
rebuild_week=args.rebuild_week.strip(),
95
+ source_run_id=args.source_run_id.strip(),
96
publish_release=args.publish_release,
97
)
98
payload = {
@@ -86,6 +100,7 @@ def main(argv: list[str] | None = None) -> int:
100
"source_refresh_policy": decision.source_refresh_policy,
101
"publish_allowed": decision.publish_allowed,
102
"crawl_allowed": decision.crawl_allowed,
103
+ "source_run_id": args.source_run_id.strip(),
104
"action": decision.action,
105
"valid": not decision.reasons,
106
"reasons": decision.reasons,
tests/test_pipeline.py
+2
-1
@@ -213,7 +213,8 @@ class WorkflowConfigTests(unittest.TestCase):
213
self.assertIn("COUNTER=$(cat .squad/run-counter.txt", run_script)
214
self.assertIn("COUNTER=$((COUNTER + 1))", run_script)
215
self.assertIn(".squad/run-counter.txt", run_script)
216
- self.assertIn("git add data/raw/ data/snapshots/ .squad/run-counter.txt", run_script)
216
+ self.assertIn("git add data/raw/ data/snapshots/ data/raw-store/", run_script)
217
+ self.assertIn("git add .squad/run-counter.txt", run_script)
218
219
def test_external_news_workflow_passes_deterministic_until(self) -> None:
220
workflow_path = Path(".github/workflows/crawl-and-publish.yml")
tests/test_publish_manifest.py
+117
@@ -140,6 +140,9 @@ def create_args(
140
gate_report: Path | None = None,
141
validation_status: str = "passed",
142
preflight: Path | None | bool = True,
143
+ run_mode: str = "normal",
144
+ source_run_id: str = "",
145
+ raw_store_manifest: Path | None = None,
146
) -> list[str]:
147
args = [
148
"create",
@@ -149,6 +152,8 @@ def create_args(
152
RUN_ID,
153
"--current-datetime",
154
CURRENT_DATETIME,
155
+ "--root",
156
+ str(base),
157
"--summary",
158
str(summary),
159
"--published-summary",
@@ -159,6 +164,8 @@ def create_args(
164
source,
165
"--validation-status",
166
validation_status,
167
+ "--run-mode",
168
+ run_mode,
169
"--output",
170
str(manifest),
171
]
@@ -172,6 +179,10 @@ def create_args(
179
args.extend(["--preflight-report", str(preflight_path)])
180
elif isinstance(preflight, Path):
181
args.extend(["--preflight-report", str(preflight)])
182
+ if source_run_id:
183
+ args.extend(["--source-run-id", source_run_id])
184
+ if raw_store_manifest is not None:
185
+ args.extend(["--raw-store-manifest", str(raw_store_manifest)])
186
return args
187
188
@@ -224,6 +235,112 @@ class PublishManifestTests(unittest.TestCase):
235
)
236
self.assertEqual(assert_eligible_from_root(base, manifest), 0)
237
238
+ def test_restore_candidate_requires_verified_source_bound_raw_store(self) -> None:
239
+ tests_root = Path(__file__).resolve().parent
240
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
241
+ base = Path(tmpdir)
242
+ raw = base / "data/raw/2026-W21.json"
243
+ summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
244
+ manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
245
+ gate_report = base / "data/candidates/2026-W21/123456/analysis-gate-report.json"
246
+ write_raw(raw)
247
+ write_summary(summary)
248
+ write_gate_report(gate_report)
249
+ publish_manifest.publish_safety.main(
250
+ [
251
+ "store-raw",
252
+ "--root",
253
+ str(base),
254
+ "--week",
255
+ WEEK,
256
+ "--source-run-id",
257
+ "26753498571",
258
+ "--source-artifact-id",
259
+ "7330965888",
260
+ "--source-head-sha",
261
+ "abc123",
262
+ "--path",
263
+ "data/raw/2026-W21.json",
264
+ ]
265
+ )
266
+ raw_store_manifest = base / "data/raw-store/2026-W21/26753498571/manifest.json"
267
+
268
+ publish_manifest.main(
269
+ create_args(
270
+ base,
271
+ raw,
272
+ summary,
273
+ manifest,
274
+ gate_report=gate_report,
275
+ run_mode="restore",
276
+ source_run_id="26753498571",
277
+ raw_store_manifest=raw_store_manifest,
278
+ )
279
+ )
280
+
281
+ payload = json.loads(manifest.read_text(encoding="utf-8"))
282
+ self.assertTrue(payload["promotion"]["eligible"])
283
+ self.assertTrue(payload["restore"]["verified"])
284
+ self.assertEqual(payload["restore"]["source_run_id"], "26753498571")
285
+ self.assertEqual(payload["restore"]["source_artifact"]["id"], "7330965888")
286
+ self.assertEqual(assert_eligible_from_root(base, manifest), 0)
287
+
288
+ def test_restore_candidate_rejects_hash_mismatch_before_acceptance(self) -> None:
289
+ tests_root = Path(__file__).resolve().parent
290
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
291
+ base = Path(tmpdir)
292
+ raw = base / "data/raw/2026-W21.json"
293
+ summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
294
+ manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
295
+ gate_report = base / "data/candidates/2026-W21/123456/analysis-gate-report.json"
296
+ write_raw(raw)
297
+ write_summary(summary)
298
+ write_gate_report(gate_report)
299
+ publish_manifest.publish_safety.main(
300
+ [
301
+ "store-raw",
302
+ "--root",
303
+ str(base),
304
+ "--week",
305
+ WEEK,
306
+ "--source-run-id",
307
+ "26753498571",
308
+ "--source-artifact-id",
309
+ "7330965888",
310
+ "--source-head-sha",
311
+ "abc123",
312
+ "--path",
313
+ "data/raw/2026-W21.json",
314
+ ]
315
+ )
316
+ raw_store_manifest = base / "data/raw-store/2026-W21/26753498571/manifest.json"
317
+ raw.write_bytes(b"tampered restored input\n")
318
+
319
+ publish_manifest.main(
320
+ create_args(
321
+ base,
322
+ raw,
323
+ summary,
324
+ manifest,
325
+ gate_report=gate_report,
326
+ run_mode="restore",
327
+ source_run_id="26753498571",
328
+ raw_store_manifest=raw_store_manifest,
329
+ )
330
+ )
331
+
332
+ payload = json.loads(manifest.read_text(encoding="utf-8"))
333
+ self.assertFalse(payload["promotion"]["eligible"])
334
+ self.assertFalse(payload["restore"]["verified"])
335
+ self.assertTrue(
336
+ any(
337
+ "Restored raw input" in reason and "mismatch" in reason
338
+ for reason in payload["promotion"]["reasons"]
339
+ )
340
+ )
341
+ with self.assertRaises(SystemExit):
342
+ assert_eligible_from_root(base, manifest)
343
+
344
def test_no_ai_candidate_is_not_eligible(self) -> None:
345
tests_root = Path(__file__).resolve().parent
346
with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
tests/test_publish_safety.py
+162
@@ -228,6 +228,168 @@ class PublishSafetyTests(unittest.TestCase):
228
finally:
229
outside_manifest.unlink(missing_ok=True)
230
231
+ def test_raw_store_is_immutable_and_restore_is_hash_verified(self) -> None:
232
+ tests_root = Path(__file__).resolve().parent
233
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
234
+ root = Path(tmpdir)
235
+ raw = root / "data/raw/2026-W23.json"
236
+ raw.parent.mkdir(parents=True, exist_ok=True)
237
+ original_raw = b'{"week":"2026-W23","evidence":"original"}\n'
238
+ raw.write_bytes(original_raw)
239
+
240
+ self.assertEqual(
241
+ publish_safety.main(
242
+ [
243
+ "store-raw",
244
+ "--root",
245
+ str(root),
246
+ "--week",
247
+ "2026-W23",
248
+ "--source-run-id",
249
+ "26753498571",
250
+ "--source-artifact-id",
251
+ "7330965888",
252
+ "--source-head-sha",
253
+ "abc123",
254
+ "--path",
255
+ "data/raw/2026-W23.json",
256
+ ]
257
+ ),
258
+ 0,
259
+ )
260
+ store = root / "data/raw-store/2026-W23/26753498571"
261
+ manifest = json.loads((store / "manifest.json").read_text(encoding="utf-8"))
262
+ self.assertEqual(manifest["schema_version"], "raw_store_v1")
263
+ self.assertEqual(manifest["source_run_id"], "26753498571")
264
+ self.assertEqual(manifest["source_artifact"]["id"], "7330965888")
265
+ self.assertEqual(manifest["source_artifact"]["retention_days"], 90)
266
+ self.assertEqual(manifest["files"][0]["original_path"], "data/raw/2026-W23.json")
267
+ original_store_bytes = {
268
+ path.relative_to(store): path.read_bytes()
269
+ for path in store.rglob("*")
270
+ if path.is_file()
271
+ }
272
+
273
+ raw.write_bytes(b"replacement that must not enter immutable storage\n")
274
+ with self.assertRaisesRegex(SystemExit, "Refusing to overwrite immutable raw store"):
275
+ publish_safety.main(
276
+ [
277
+ "store-raw",
278
+ "--root",
279
+ str(root),
280
+ "--week",
281
+ "2026-W23",
282
+ "--source-run-id",
283
+ "26753498571",
284
+ "--source-artifact-id",
285
+ "7330965888",
286
+ "--source-head-sha",
287
+ "abc123",
288
+ "--path",
289
+ "data/raw/2026-W23.json",
290
+ ]
291
+ )
292
+ self.assertEqual(
293
+ original_store_bytes,
294
+ {
295
+ path.relative_to(store): path.read_bytes()
296
+ for path in store.rglob("*")
297
+ if path.is_file()
298
+ },
299
+ )
300
+
301
+ stale_optional = raw.parent / "2026-W23-external-news.json"
302
+ stale_optional.write_bytes(b"stale evidence from a different source run\n")
303
+ self.assertEqual(
304
+ publish_safety.main(
305
+ [
306
+ "restore-raw",
307
+ "--root",
308
+ str(root),
309
+ "--week",
310
+ "2026-W23",
311
+ "--source-run-id",
312
+ "26753498571",
313
+ ]
314
+ ),
315
+ 0,
316
+ )
317
+ self.assertEqual(raw.read_bytes(), original_raw)
318
+ self.assertFalse(stale_optional.exists())
319
+
320
+ stored_raw = store / "files/data/raw/2026-W23.json"
321
+ stored_raw.write_bytes(b"X" * len(original_raw))
322
+ raw.write_bytes(b"accepted input must remain unchanged\n")
323
+ stale_on_failure = raw.parent / "2026-W23-techcrunch.json"
324
+ stale_on_failure.write_bytes(b"also unchanged when stored verification fails\n")
325
+ with self.assertRaisesRegex(SystemExit, "checksum mismatch"):
326
+ publish_safety.main(
327
+ [
328
+ "restore-raw",
329
+ "--root",
330
+ str(root),
331
+ "--week",
332
+ "2026-W23",
333
+ "--source-run-id",
334
+ "26753498571",
335
+ ]
336
+ )
337
+ self.assertEqual(raw.read_bytes(), b"accepted input must remain unchanged\n")
338
+ self.assertTrue(stale_on_failure.exists())
339
+
340
+ def test_restore_accepts_custom_source_artifact_name(self) -> None:
341
+ """A non-default --source-artifact-name is still valid provenance and restorable."""
342
+ tests_root = Path(__file__).resolve().parent
343
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
344
+ root = Path(tmpdir)
345
+ raw = root / "data/raw/2026-W24.json"
346
+ raw.parent.mkdir(parents=True, exist_ok=True)
347
+ original_raw = b'{"week":"2026-W24","evidence":"original"}\n'
348
+ raw.write_bytes(original_raw)
349
+
350
+ self.assertEqual(
351
+ publish_safety.main(
352
+ [
353
+ "store-raw",
354
+ "--root",
355
+ str(root),
356
+ "--week",
357
+ "2026-W24",
358
+ "--source-run-id",
359
+ "26753498572",
360
+ "--source-artifact-id",
361
+ "7330965999",
362
+ "--source-artifact-name",
363
+ "raw-evidence-bundle",
364
+ "--source-head-sha",
365
+ "def456",
366
+ "--path",
367
+ "data/raw/2026-W24.json",
368
+ ]
369
+ ),
370
+ 0,
371
+ )
372
+ store = root / "data/raw-store/2026-W24/26753498572"
373
+ manifest = json.loads((store / "manifest.json").read_text(encoding="utf-8"))
374
+ self.assertEqual(manifest["source_artifact"]["name"], "raw-evidence-bundle")
375
+
376
+ raw.write_bytes(b"drift that restore must overwrite\n")
377
+ self.assertEqual(
378
+ publish_safety.main(
379
+ [
380
+ "restore-raw",
381
+ "--root",
382
+ str(root),
383
+ "--week",
384
+ "2026-W24",
385
+ "--source-run-id",
386
+ "26753498572",
387
+ ]
388
+ ),
389
+ 0,
390
+ )
391
+ self.assertEqual(raw.read_bytes(), original_raw)
392
+
393
394
if __name__ == "__main__":
395
unittest.main()
tests/test_rerun_modes.py
+32
@@ -21,6 +21,38 @@ class RerunModeTests(unittest.TestCase):
21
22
self.assertIn("requires run_mode=restore", decision.reasons[0])
23
24
+ def test_restore_requires_source_run_id(self) -> None:
25
+ decision = validate_modes(
26
+ run_mode="restore",
27
+ source_refresh_policy="reuse-same-day",
28
+ rebuild_week="2026-W21",
29
+ )
30
+
31
+ self.assertTrue(any("requires source_run_id" in reason for reason in decision.reasons))
32
+
33
+ def test_restore_accepts_source_bound_week(self) -> None:
34
+ decision = validate_modes(
35
+ run_mode="restore",
36
+ source_refresh_policy="reuse-same-day",
37
+ rebuild_week="2026-W21",
38
+ source_run_id="26753498571",
39
+ )
40
+
41
+ self.assertFalse(decision.reasons)
42
+ self.assertFalse(decision.crawl_allowed)
43
+ self.assertTrue(decision.publish_allowed)
44
+
45
+ def test_source_run_id_is_restore_only(self) -> None:
46
+ decision = validate_modes(
47
+ run_mode="normal",
48
+ source_refresh_policy="reuse-same-day",
49
+ source_run_id="26753498571",
50
+ )
51
+
52
+ self.assertTrue(
53
+ any("only allowed with run_mode=restore" in reason for reason in decision.reasons)
54
+ )
55
+
56
def test_dry_run_cannot_publish_release(self) -> None:
57
decision = validate_modes(
58
run_mode="dry-run",
tests/test_sync_publish_workflow.py
+31
@@ -2,6 +2,7 @@ from pathlib import Path
2
3
WORKFLOW = Path(".github/workflows/sync-publish-to-main.yml")
4
RESTORE_WORKFLOW = Path(".github/workflows/restore-publish-backup.yml")
5
+CRAWL_WORKFLOW = Path(".github/workflows/crawl-and-publish.yml")
6
7
8
def test_publish_sync_only_checks_out_generated_content_paths() -> None:
@@ -21,6 +22,7 @@ def test_publish_sync_only_checks_out_generated_content_paths() -> None:
22
assert "git ls-tree -r --name-only origin/publish -- .squad" not in workflow
23
assert "squad learnings" not in workflow.lower()
24
assert "python3 scripts/generate_rollups.py" in workflow
25
+ assert "data/raw-store/" not in workflow
26
27
28
def test_publish_sync_refuses_staged_squad_changes() -> None:
@@ -50,3 +52,32 @@ def test_restore_publish_backup_workflow_keeps_helper_available_after_publish_ch
52
assert "cd publish" in workflow
53
assert "python3 scripts/publish_safety.py restore-backup" not in workflow
54
assert "python3 ../workflow-source/scripts/publish_safety.py restore-backup" in workflow
55
+
56
+
57
+def test_publish_sync_stages_before_cached_diff_evaluation() -> None:
58
+ workflow = WORKFLOW.read_text(encoding="utf-8")
59
+
60
+ assert workflow.index("git add -A") < workflow.index("if git diff --cached --quiet; then")
61
+
62
+
63
+def test_crawl_workflow_stores_and_restores_source_bound_raw_evidence() -> None:
64
+ workflow = CRAWL_WORKFLOW.read_text(encoding="utf-8")
65
+
66
+ assert "source_run_id:" in workflow
67
+ assert "run_mode=restore and source_run_id" in workflow
68
+ assert workflow.count("python3 scripts/publish_safety.py restore-raw") == 2
69
+ assert "python3 publish-safety-tool.py store-raw" in workflow
70
+ assert 'RAW_STORE_DIR="data/raw-store/${REBUILD_WEEK}/${SOURCE_RUN_ID}"' in workflow
71
+ assert (
72
+ '--raw-store-manifest "data/raw-store/${WEEK}/${SOURCE_RUN_ID}/manifest.json"' in workflow
73
+ )
74
+ assert 'source-artifact-id "$RAW_ARTIFACT_ID"' in workflow
75
+ assert "retention-days: 90" in workflow
76
+
77
+
78
+def test_crawl_workflow_stages_raw_store_before_cached_diff_evaluation() -> None:
79
+ workflow = CRAWL_WORKFLOW.read_text(encoding="utf-8")
80
+
81
+ stage = workflow.index("git add data/raw/ data/snapshots/ data/raw-store/")
82
+ cached_diff = workflow.index("if git diff --cached --quiet; then", stage)
83
+ assert stage < cached_diff