@cryptotaxi247 / SquadScope / commits / 1de643b

Add publish backups and concurrency safeguards (#280)

* fix: harden publish backups and concurrency Create immutable publish backups before weekly replacements, record source provenance for restore audits, and add expected-ref safeguards around publish branch promotions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: restore publish backup helper availability Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: harden publish backup safety Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Juan Manuel Servera committed Jun 6, 2026 at 22:04 UTC 1de643b0a73821b8dfb5295cfcd8ec10ea6b1bb3

8 files changed +588 -12

.github/workflows/crawl-and-publish.yml

+84 -10

@@ -247,8 +247,10 @@ jobs:

247 cp -r data/snapshots crawl-snapshots-backup

248 # Fetch or create the unprotected data branch

249 if git fetch origin "$DATA_BRANCH" 2>/dev/null; then

250 + EXPECTED_PUBLISH_SHA=$(git rev-parse "origin/$DATA_BRANCH")

251 git checkout -f -B "$DATA_BRANCH" "origin/$DATA_BRANCH"

252 else

253 + EXPECTED_PUBLISH_SHA=""

254 git checkout -f -B "$DATA_BRANCH" "origin/$DEFAULT_BRANCH"

255 fi

256 # Restore crawl output on top of data branch

@@ -267,7 +269,11 @@ jobs:

269 git add data/raw/ data/snapshots/ .squad/run-counter.txt

270 git diff --cached --quiet && exit 0

271 git commit -m "data: weekly crawl $WEEK [run #${GITHUB_RUN_ID}]"

270 - git push origin "$DATA_BRANCH"

272 + if [ -n "$EXPECTED_PUBLISH_SHA" ]; then

273 + git push --force-with-lease="refs/heads/$DATA_BRANCH:$EXPECTED_PUBLISH_SHA" origin HEAD:"$DATA_BRANCH"

274 + else

275 + git push origin HEAD:"$DATA_BRANCH"

276 + fi

277

278 analyze:

279 needs: crawl

@@ -282,6 +288,8 @@ jobs:

288 candidate_summary_file: ${{ steps.analysis-context.outputs.candidate_output_file }}

289 publish_manifest_file: ${{ steps.analysis-context.outputs.publish_manifest_file }}

290 current_datetime: ${{ steps.analysis-context.outputs.current_datetime }}

291 + publish_base_sha: ${{ steps.publish-base.outputs.sha }}

292 + publish_head_sha: ${{ steps.commit-analysis.outputs.publish_head_sha || steps.publish-base.outputs.sha }}

293 run_mode: ${{ steps.analysis-context.outputs.run_mode }}

294

295 steps:

@@ -319,6 +327,16 @@ jobs:

327 git checkout origin/publish -- "$f" 2>/dev/null || true

328 done

329

330 + - name: Record publish branch base

331 + id: publish-base

332 + run: |

333 + set -euo pipefail

334 + if git fetch origin publish 2>/dev/null; then

335 + echo "sha=$(git rev-parse origin/publish)" >> "$GITHUB_OUTPUT"

336 + else

337 + echo "sha=" >> "$GITHUB_OUTPUT"

338 + fi

339 +

340 - name: Set up Node

341 uses: actions/setup-node@v4

342 with:

@@ -716,6 +734,7 @@ jobs:

734 run: python3 scripts/publish_manifest.py assert-eligible --manifest "${{ steps.analysis-context.outputs.publish_manifest_file }}"

735

736 - name: Commit analysis and learnings to data branch

737 + id: commit-analysis

738 if: ${{ steps.analysis-context.outputs.run_mode != 'dry-run' && steps.analysis-context.outputs.run_mode != 'candidate-only' }}

739 env:

740 DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}

@@ -724,17 +743,19 @@ jobs:

743 CANDIDATE_SUMMARY: ${{ steps.analysis-context.outputs.candidate_output_file }}

744 PUBLISHED_SUMMARY: ${{ steps.analysis-context.outputs.published_output_file }}

745 MANIFEST_FILE: ${{ steps.analysis-context.outputs.publish_manifest_file }}

746 + EXPECTED_PUBLISH_SHA: ${{ steps.publish-base.outputs.sha }}

747 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

748 run: |

749 set -euo pipefail

750 python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE"

751 cp scripts/publish_manifest.py publish-manifest-tool.py

752 + cp scripts/publish_safety.py publish-safety-tool.py

753 git config user.name "github-actions[bot]"

754 git config user.email "github-actions[bot]@users.noreply.github.com"

755 # Check for analysis data OR squad learning state changes

756 if ! git status --short -- data/analyzed data/candidates data/metrics .squad | grep -q .; then

757 echo "No analyzed data, token usage, or learning state changes to commit."

737 - rm -f publish-manifest-tool.py

758 + rm -f publish-manifest-tool.py publish-safety-tool.py

759 exit 0

760 fi

761 mkdir -p "$(dirname "$PUBLISHED_SUMMARY")"

@@ -751,31 +772,59 @@ jobs:

772 fi

773 # Push to the unprotected data branch

774 if git fetch origin "$DATA_BRANCH" 2>/dev/null; then

775 + CURRENT_PUBLISH_SHA=$(git rev-parse "origin/$DATA_BRANCH")

776 + if [ -n "$EXPECTED_PUBLISH_SHA" ] && [ "$CURRENT_PUBLISH_SHA" != "$EXPECTED_PUBLISH_SHA" ]; then

777 + echo "::error::Publish branch drifted since analysis began: expected $EXPECTED_PUBLISH_SHA, found $CURRENT_PUBLISH_SHA"

778 + exit 1

779 + fi

780 git checkout -f -B "$DATA_BRANCH" "origin/$DATA_BRANCH"

781 else

782 + CURRENT_PUBLISH_SHA=""

783 + if [ -n "$EXPECTED_PUBLISH_SHA" ]; then

784 + echo "::error::Publish branch disappeared since analysis began: expected $EXPECTED_PUBLISH_SHA"

785 + exit 1

786 + fi

787 git fetch origin "$DEFAULT_BRANCH"

788 git checkout -f -B "$DATA_BRANCH" "origin/$DEFAULT_BRANCH"

789 fi

790 mkdir -p data/analyzed data/candidates data/metrics

791 python3 publish-manifest-tool.py assert-eligible --manifest "candidates-data-backup/${WEEK}/${GITHUB_RUN_ID}/publish-manifest.json"

792 + cp -r "candidates-data-backup/${WEEK}" data/candidates/

793 + python3 publish-safety-tool.py backup-existing \

794 + --week "$WEEK" \

795 + --run-id "$GITHUB_RUN_ID" \

796 + --kind analysis \

797 + --manifest "data/candidates/${WEEK}/${GITHUB_RUN_ID}/publish-manifest.json" \

798 + --expected-publish-ref "$EXPECTED_PUBLISH_SHA" \

799 + --actual-publish-ref "$CURRENT_PUBLISH_SHA" \

800 + --path "data/analyzed/${WEEK}-summary.md" \

801 + --path "data/analyzed/${WEEK}-correlations.json" \

802 + --path "data/analyzed/${WEEK}-press-context.md"

803 # Only copy current week's analysis files — do not overwrite prior weeks

804 # Required: current week's promoted summary must be backed by an eligible manifest

805 cp "analyzed-data-backup/${WEEK}-summary.md" data/analyzed/

764 - cp -r "candidates-data-backup/${WEEK}" data/candidates/

806 # Optional sidecars: conditionally generated

807 cp "analyzed-data-backup/${WEEK}-correlations.json" data/analyzed/ 2>/dev/null || true

808 cp "analyzed-data-backup/${WEEK}-press-context.md" data/analyzed/ 2>/dev/null || true

809 cp -r metrics-data-backup/* data/metrics/ 2>/dev/null || true

769 - rm -rf analyzed-data-backup candidates-data-backup metrics-data-backup publish-manifest-tool.py

810 + rm -rf analyzed-data-backup candidates-data-backup metrics-data-backup publish-manifest-tool.py publish-safety-tool.py

811 # Restore squad learning state

812 if [ "$HAS_LEARNINGS" = "true" ]; then

813 cp -r squad-learning-backup/* .squad/ 2>/dev/null || true

814 rm -rf squad-learning-backup

815 fi

775 - git add data/analyzed/ data/candidates/ data/metrics/ .squad/

776 - git diff --cached --quiet && exit 0

816 + git add data/analyzed/ data/candidates/ data/metrics/ data/backups/ .squad/

817 + if git diff --cached --quiet; then

818 + echo "publish_head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

819 + exit 0

820 + fi

821 git commit -m "analysis: weekly summary + learnings $WEEK [run #${GITHUB_RUN_ID}]"

778 - git push origin "$DATA_BRANCH"

822 + if [ -n "$CURRENT_PUBLISH_SHA" ]; then

823 + git push --force-with-lease="refs/heads/$DATA_BRANCH:$CURRENT_PUBLISH_SHA" origin HEAD:"$DATA_BRANCH"

824 + else

825 + git push origin HEAD:"$DATA_BRANCH"

826 + fi

827 + echo "publish_head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

828

829 - name: Upload analyzed data

830 if: always()

@@ -877,14 +926,17 @@ jobs:

926 WEEK: ${{ needs.analyze.outputs.week }}

927 PAGE_PATH: ${{ steps.generate-content.outputs.page_path }}

928 MANIFEST_FILE: ${{ needs.analyze.outputs.publish_manifest_file }}

929 + EXPECTED_PUBLISH_SHA: ${{ needs.analyze.outputs.publish_head_sha }}

930 GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

931 run: |

932 set -euo pipefail

933 python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE"

934 + cp scripts/publish_safety.py publish-safety-tool.py

935 git config user.name "github-actions[bot]"

936 git config user.email "github-actions[bot]@users.noreply.github.com"

937 if ! git status --short -- content/weekly content/monthly content/yearly | grep -q .; then

938 echo "No generated content changes to commit."

939 + rm -f publish-safety-tool.py

940 exit 0

941 fi

942 cp -r content/weekly content-weekly-backup 2>/dev/null || true

@@ -892,8 +944,18 @@ jobs:

944 cp -r content/yearly content-yearly-backup 2>/dev/null || true

945 # Push to the unprotected data branch

946 if git fetch origin "$DATA_BRANCH" 2>/dev/null; then

947 + CURRENT_PUBLISH_SHA=$(git rev-parse "origin/$DATA_BRANCH")

948 + if [ -n "$EXPECTED_PUBLISH_SHA" ] && [ "$CURRENT_PUBLISH_SHA" != "$EXPECTED_PUBLISH_SHA" ]; then

949 + echo "::error::Publish branch drifted between analyze and content promotion: expected $EXPECTED_PUBLISH_SHA, found $CURRENT_PUBLISH_SHA"

950 + exit 1

951 + fi

952 git checkout -f -B "$DATA_BRANCH" "origin/$DATA_BRANCH"

953 else

954 + CURRENT_PUBLISH_SHA=""

955 + if [ -n "$EXPECTED_PUBLISH_SHA" ]; then

956 + echo "::error::Publish branch disappeared between analyze and content promotion: expected $EXPECTED_PUBLISH_SHA"

957 + exit 1

958 + fi

959 git fetch origin "$DEFAULT_BRANCH"

960 git checkout -f -B "$DATA_BRANCH" "origin/$DEFAULT_BRANCH"

961 fi

@@ -913,16 +975,28 @@ jobs:

975 esac

976 RELATIVE_FROM_WEEKLY="${PAGE_PATH#content/weekly/}"

977 mkdir -p "$(dirname "$PAGE_PATH")"

978 + python3 publish-safety-tool.py backup-existing \

979 + --week "$WEEK" \

980 + --run-id "$GITHUB_RUN_ID" \

981 + --kind content \

982 + --manifest "$MANIFEST_FILE" \

983 + --expected-publish-ref "$EXPECTED_PUBLISH_SHA" \

984 + --actual-publish-ref "$CURRENT_PUBLISH_SHA" \

985 + --path "$PAGE_PATH"

986 cp "content-weekly-backup/${RELATIVE_FROM_WEEKLY}" "$PAGE_PATH"

987 # Monthly/yearly rollups are safe to copy since generate_rollups.py was seeded

988 # with hydrated prior-week data (see Hydrate analyzed data step above)

989 cp -r content-monthly-backup/* content/monthly/ 2>/dev/null || true

990 cp -r content-yearly-backup/* content/yearly/ 2>/dev/null || true

921 - rm -rf content-weekly-backup content-monthly-backup content-yearly-backup

922 - git add content/weekly/ content/monthly/ content/yearly/

991 + rm -rf content-weekly-backup content-monthly-backup content-yearly-backup publish-safety-tool.py

992 + git add content/weekly/ content/monthly/ content/yearly/ data/backups/

993 git diff --cached --quiet && exit 0

994 git commit -m "content: weekly page $WEEK [run #${GITHUB_RUN_ID}]"

925 - git push origin "$DATA_BRANCH"

995 + if [ -n "$CURRENT_PUBLISH_SHA" ]; then

996 + git push --force-with-lease="refs/heads/$DATA_BRANCH:$CURRENT_PUBLISH_SHA" origin HEAD:"$DATA_BRANCH"

997 + else

998 + git push origin HEAD:"$DATA_BRANCH"

999 + fi

1000

1001 - name: Upload generated content artifact

1002 uses: actions/upload-artifact@v4

.github/workflows/restore-publish-backup.yml new

+53

@@ -0,0 +1,53 @@

1 +name: Restore publish backup

2 +

3 +on:

4 + workflow_dispatch:

5 + inputs:

6 + backup_manifest:

7 + description: 'Immutable backup manifest on publish (for example data/backups/2026-W23/123/content/manifest.json).'

8 + required: true

9 + type: string

10 +

11 +permissions:

12 + contents: write

13 +

14 +concurrency:

15 + group: restore-publish-backup

16 + cancel-in-progress: false

17 +

18 +jobs:

19 + restore:

20 + runs-on: ubuntu-latest

21 + steps:

22 + - name: Check out workflow source

23 + uses: actions/checkout@v4

24 + with:

25 + ref: ${{ github.sha }}

26 + path: workflow-source

27 +

28 + - name: Check out publish

29 + uses: actions/checkout@v4

30 + with:

31 + ref: publish

32 + fetch-depth: 0

33 + path: publish

34 +

35 + - name: Restore immutable backup

36 + env:

37 + BACKUP_MANIFEST: ${{ inputs.backup_manifest }}

38 + run: |

39 + set -euo pipefail

40 + cd publish

41 + git config user.name "github-actions[bot]"

42 + git config user.email "github-actions[bot]@users.noreply.github.com"

43 + git fetch origin publish

44 + EXPECTED_PUBLISH_SHA=$(git rev-parse origin/publish)

45 + git checkout -f -B publish origin/publish

46 + python3 ../workflow-source/scripts/publish_safety.py restore-backup --backup-manifest "$BACKUP_MANIFEST"

47 + git add data/analyzed/ content/weekly/

48 + if git diff --cached --quiet; then

49 + echo "Backup restore produced no changes."

50 + exit 0

51 + fi

52 + git commit -m "restore: publish backup ${BACKUP_MANIFEST}"

53 + git push --force-with-lease="refs/heads/publish:$EXPECTED_PUBLISH_SHA" origin HEAD:publish

scripts/publish_manifest.py

+16 -2

@@ -319,17 +319,26 @@ def artifact_entry(

319 artifact_generated_at = payload.get("generated_at") or payload.get("crawled_at") or generated_at

320 else:

321 artifact_generated_at = generated_at

322 + reuse_status = same_day_reuse_status(payload)

323 + checksum = sha256_file(path)

324 entry: dict[str, Any] = {

325 "role": role,

326 "path": path.as_posix(),

327 "exists": path.exists(),

328 "size_bytes": path.stat().st_size if path.exists() else 0,

327 - "sha256": sha256_file(path),

329 + "sha256": checksum,

330 "artifact_checksum": metadata.get("artifact_checksum"),

331 "week": payload.get("week") if isinstance(payload, dict) else None,

332 "crawled_at": payload.get("crawled_at") if isinstance(payload, dict) else None,

333 "generated_at": artifact_generated_at,

332 - "same_day_reuse": same_day_reuse_status(payload),

334 + "same_day_reuse": reuse_status,

335 + "provenance": {

336 + "path": path.as_posix(),

337 + "sha256": checksum,

338 + "artifact_checksum": metadata.get("artifact_checksum"),

339 + "generated_at": artifact_generated_at,

340 + "same_day_reuse": reuse_status,

341 + },

342 "freshness": freshness_for_json_artifact(role, week, payload, run_date=run_date, run_mode=run_mode) if path.suffix == ".json" else {"status": "not_applicable", "reasons": []},

343 }

344 if "source_status" in metadata:

@@ -701,6 +710,11 @@ def assert_eligible(args: argparse.Namespace) -> int:

710 for entry in source_artifacts:

711 if not isinstance(entry, dict) or not entry.get("sha256"):

712 raise SystemExit("Manifest source artifact is missing a checksum.")

713 + provenance = entry.get("provenance")

714 + if not isinstance(provenance, dict) or provenance.get("sha256") != entry.get("sha256"):

715 + raise SystemExit("Manifest source artifact is missing auditable provenance.")

716 + if not isinstance(provenance.get("same_day_reuse"), dict):

717 + raise SystemExit("Manifest source artifact reuse provenance is missing.")

718 freshness = entry.get("freshness", {})

719 if isinstance(freshness, dict) and freshness.get("status") == "stale":

720 raise SystemExit(f"Manifest source artifact is stale: {entry.get('path')}")

scripts/publish_safety.py new

+194

@@ -0,0 +1,194 @@

1 +#!/usr/bin/env python3

2 +from __future__ import annotations

3 +

4 +import argparse

5 +import hashlib

6 +import json

7 +import shutil

8 +from datetime import UTC, datetime

9 +from pathlib import Path

10 +from typing import Any

11 +

12 +

13 +BACKUP_SCHEMA_VERSION = "publish_backup_v1"

14 +PUBLISH_MANIFEST_SCHEMA_VERSION = "publish_eligibility_v1"

15 +

16 +

17 +def sha256_file(path: Path) -> str | None:

18 + if not path.exists() or not path.is_file():

19 + return None

20 + digest = hashlib.sha256()

21 + with path.open("rb") as handle:

22 + for chunk in iter(lambda: handle.read(1024 * 1024), b""):

23 + digest.update(chunk)

24 + return digest.hexdigest()

25 +

26 +

27 +def load_json(path: Path) -> dict[str, Any] | None:

28 + try:

29 + payload = json.loads(path.read_text(encoding="utf-8"))

30 + except (FileNotFoundError, OSError, json.JSONDecodeError):

31 + return None

32 + return payload if isinstance(payload, dict) else None

33 +

34 +

35 +def relpath_under_root(root: Path, value: str) -> Path:

36 + path = Path(value)

37 + if path.is_absolute():

38 + raise SystemExit(f"Path must be relative to repository root: {value}")

39 + resolved_root = root.resolve()

40 + resolved_path = (resolved_root / path).resolve()

41 + try:

42 + return resolved_path.relative_to(resolved_root)

43 + except ValueError as exc:

44 + raise SystemExit(f"Path must stay under repository root: {value}") from exc

45 +

46 +

47 +def path_under_root(root: Path, value: Path) -> tuple[Path, Path]:

48 + resolved_root = root.resolve()

49 + resolved_path = (value if value.is_absolute() else resolved_root / value).resolve()

50 + try:

51 + relative = resolved_path.relative_to(resolved_root)

52 + except ValueError as exc:

53 + raise SystemExit(f"Path must stay under repository root: {value}") from exc

54 + return relative, resolved_path

55 +

56 +

57 +def parse_args(argv: list[str] | None = None) -> argparse.Namespace:

58 + parser = argparse.ArgumentParser(description="Publish-branch backup and restore safeguards.")

59 + subparsers = parser.add_subparsers(dest="command", required=True)

60 +

61 + backup = subparsers.add_parser("backup-existing", help="Create an immutable backup manifest for target paths.")

62 + backup.add_argument("--root", default=".", type=Path)

63 + backup.add_argument("--week", required=True)

64 + backup.add_argument("--run-id", required=True)

65 + backup.add_argument("--kind", required=True, choices=["analysis", "content"])

66 + backup.add_argument("--manifest", required=True, type=Path, help="Publish eligibility manifest.")

67 + backup.add_argument("--expected-publish-ref", default="")

68 + backup.add_argument("--actual-publish-ref", default="")

69 + backup.add_argument("--backup-root", default=Path("data/backups"), type=Path)

70 + backup.add_argument("--path", action="append", required=True, help="Published path to snapshot before replacement.")

71 +

72 + restore = subparsers.add_parser("restore-backup", help="Restore files from an immutable publish backup manifest.")

73 + restore.add_argument("--root", default=".", type=Path)

74 + restore.add_argument("--backup-manifest", required=True, type=Path)

75 +

76 + return parser.parse_args(argv)

77 +

78 +

79 +def backup_existing(args: argparse.Namespace) -> int:

80 + root = args.root.resolve()

81 + source_manifest_relative = relpath_under_root(root, args.manifest.as_posix())

82 + source_manifest = root / source_manifest_relative

83 + source_manifest_payload = load_json(source_manifest)

84 + if source_manifest_payload is None:

85 + raise SystemExit(f"Publish manifest is missing or malformed: {source_manifest_relative.as_posix()}")

86 + if source_manifest_payload.get("schema_version") != PUBLISH_MANIFEST_SCHEMA_VERSION:

87 + raise SystemExit(f"Unsupported publish manifest schema: {source_manifest_payload.get('schema_version')!r}")

88 + if not isinstance(source_manifest_payload.get("candidate"), dict):

89 + raise SystemExit("Publish manifest lacks candidate block.")

90 + if not isinstance(source_manifest_payload.get("source_artifacts"), list):

91 + raise SystemExit("Publish manifest lacks source artifact provenance.")

92 + if not isinstance(source_manifest_payload.get("analysis"), dict):

93 + raise SystemExit("Publish manifest lacks analysis provenance.")

94 +

95 + entries: list[dict[str, Any]] = []

96 + for raw_path in args.path:

97 + relative = relpath_under_root(root, raw_path)

98 + source = root / relative

99 + if source.exists() and not source.is_file():

100 + raise SystemExit(f"Backup target must be a regular file: {relative.as_posix()}")

101 + entries.append(

102 + {

103 + "path": relative.as_posix(),

104 + "existed": source.exists(),

105 + "size_bytes": source.stat().st_size if source.exists() else 0,

106 + "sha256": sha256_file(source),

107 + }

108 + )

109 +

110 + backup_root = root / relpath_under_root(root, args.backup_root.as_posix())

111 + backup_dir = backup_root / args.week / args.run_id / args.kind

112 + manifest_path = backup_dir / "manifest.json"

113 + if manifest_path.exists():

114 + raise SystemExit(f"Refusing to overwrite immutable backup manifest: {manifest_path}")

115 + backup_dir.mkdir(parents=True, exist_ok=False)

116 +

117 + files_dir = backup_dir / "files"

118 + for entry in entries:

119 + relative = Path(entry["path"])

120 + source = root / relative

121 + if source.exists():

122 + destination = files_dir / relative

123 + destination.parent.mkdir(parents=True, exist_ok=True)

124 + shutil.copy2(source, destination)

125 + entry["backup_path"] = destination.relative_to(root).as_posix()

126 +

127 + manifest = {

128 + "schema_version": BACKUP_SCHEMA_VERSION,

129 + "created_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),

130 + "week": args.week,

131 + "run_id": args.run_id,

132 + "kind": args.kind,

133 + "publish_ref": {

134 + "expected": args.expected_publish_ref,

135 + "actual": args.actual_publish_ref,

136 + },

137 + "source_manifest": {

138 + "path": source_manifest_relative.as_posix(),

139 + "sha256": sha256_file(source_manifest),

140 + "candidate": source_manifest_payload.get("candidate") if source_manifest_payload else None,

141 + "source_artifacts": source_manifest_payload.get("source_artifacts") if source_manifest_payload else None,

142 + "analysis": source_manifest_payload.get("analysis") if source_manifest_payload else None,

143 + },

144 + "files": entries,

145 + }

146 + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")

147 + print(f"Created immutable publish backup: {manifest_path.relative_to(root).as_posix()}")

148 + return 0

149 +

150 +

151 +def restore_backup(args: argparse.Namespace) -> int:

152 + root = args.root.resolve()

153 + backup_manifest_relative, backup_manifest = path_under_root(root, args.backup_manifest)

154 + payload = load_json(backup_manifest)

155 + if payload is None:

156 + raise SystemExit(f"Backup manifest is missing or malformed: {backup_manifest}")

157 + if payload.get("schema_version") != BACKUP_SCHEMA_VERSION:

158 + raise SystemExit(f"Unsupported backup schema: {payload.get('schema_version')!r}")

159 + files = payload.get("files")

160 + if not isinstance(files, list):

161 + raise SystemExit("Backup manifest has no files list.")

162 +

163 + for entry in files:

164 + if not isinstance(entry, dict) or not isinstance(entry.get("path"), str):

165 + raise SystemExit("Backup file entry is malformed.")

166 + target_relative = relpath_under_root(root, entry["path"])

167 + target = root / target_relative

168 + if entry.get("existed") is True:

169 + backup_path = entry.get("backup_path")

170 + if not isinstance(backup_path, str):

171 + raise SystemExit(f"Backup entry lacks backup_path for {entry['path']}")

172 + source_relative = relpath_under_root(root, backup_path)

173 + source = root / source_relative

174 + if sha256_file(source) != entry.get("sha256"):

175 + raise SystemExit(f"Backup checksum mismatch for {backup_path}")

176 + target.parent.mkdir(parents=True, exist_ok=True)

177 + shutil.copy2(source, target)

178 + else:

179 + target.unlink(missing_ok=True)

180 + print(f"Restored publish backup: {backup_manifest_relative.as_posix()}")

181 + return 0

182 +

183 +

184 +def main(argv: list[str] | None = None) -> int:

185 + args = parse_args(argv)

186 + if args.command == "backup-existing":

187 + return backup_existing(args)

188 + if args.command == "restore-backup":

189 + return restore_backup(args)

190 + raise AssertionError(args.command)

191 +

192 +

193 +if __name__ == "__main__":

194 + raise SystemExit(main())

tests/test_pipeline.py

+12

@@ -399,6 +399,11 @@ class WorkflowConfigTests(unittest.TestCase):

399 self.assertIsNotNone(commit_step)

400 commit_run = commit_step["run"]

401 self.assertIn('assert-eligible --manifest "$MANIFEST_FILE"', commit_run)

402 + self.assertIn("Publish branch drifted since analysis began", commit_run)

403 + self.assertIn("publish_safety.py", commit_run)

404 + self.assertIn("backup-existing", commit_run)

405 + self.assertIn("data/backups/", commit_run)

406 + self.assertIn("--force-with-lease", commit_run)

407 self.assertIn('cp "$CANDIDATE_SUMMARY" "$PUBLISHED_SUMMARY"', commit_run)

408 self.assertIn("git add data/analyzed/ data/candidates/", commit_run)

409

@@ -411,6 +416,13 @@ class WorkflowConfigTests(unittest.TestCase):

416 self.assertIsNotNone(generate_step)

417 self.assertIn('assert-eligible --manifest "$MANIFEST_FILE"', generate_step["run"])

418

419 + content_commit_step = next((s for s in generate["steps"] if s.get("name") == "Commit generated content to data branch"), None)

420 + self.assertIsNotNone(content_commit_step)

421 + content_commit_run = content_commit_step["run"]

422 + self.assertIn("Publish branch drifted between analyze and content promotion", content_commit_run)

423 + self.assertIn("backup-existing", content_commit_run)

424 + self.assertIn("--force-with-lease", content_commit_run)

425 +

426 def test_rerun_mode_inputs_and_guards_are_declared(self) -> None:

427 workflow_path = Path(".github/workflows/crawl-and-publish.yml")

428 workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8"))

tests/test_publish_manifest.py

@@ -164,6 +164,8 @@ class PublishManifestTests(unittest.TestCase):

164 self.assertEqual(payload["promotion"]["decision"], "promote")

165 self.assertRegex(payload["candidate"]["summary_sha256"], r"^[0-9a-f]{64}$")

166 self.assertRegex(payload["source_artifacts"][0]["sha256"], r"^[0-9a-f]{64}$")

167 + self.assertEqual(payload["source_artifacts"][0]["provenance"]["sha256"], payload["source_artifacts"][0]["sha256"])

168 + self.assertEqual(payload["source_artifacts"][0]["provenance"]["same_day_reuse"]["status"], "not_reused")

169 self.assertEqual(assert_eligible_from_root(base, manifest), 0)

170

171 def test_no_ai_candidate_is_not_eligible(self) -> None:

tests/test_publish_safety.py new

+206

@@ -0,0 +1,206 @@

1 +import json

2 +import tempfile

3 +import unittest

4 +from pathlib import Path

5 +

6 +from scripts import publish_safety

7 +

8 +

9 +class PublishSafetyTests(unittest.TestCase):

10 + def write_manifest(self, root: Path) -> Path:

11 + manifest = root / "data/candidates/2026-W23/99/publish-manifest.json"

12 + manifest.parent.mkdir(parents=True, exist_ok=True)

13 + manifest.write_text(

14 + json.dumps(

15 + {

16 + "schema_version": "publish_eligibility_v1",

17 + "candidate": {"summary_sha256": "candidate-sha"},

18 + "source_artifacts": [{"path": "data/raw/2026-W23.json", "sha256": "raw-sha"}],

19 + "analysis": {"source": "copilot-cli"},

20 + }

21 + ),

22 + encoding="utf-8",

23 + )

24 + return manifest

25 +

26 + def test_backup_existing_is_immutable_and_restorable_with_provenance(self) -> None:

27 + tests_root = Path(__file__).resolve().parent

28 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

29 + root = Path(tmpdir)

30 + target = root / "content/weekly/2026/W23.md"

31 + target.parent.mkdir(parents=True, exist_ok=True)

32 + target.write_text("known good article\n", encoding="utf-8")

33 + source = root / "data/raw/2026-W23.json"

34 + source.parent.mkdir(parents=True, exist_ok=True)

35 + source.write_text('{"week":"2026-W23"}\n', encoding="utf-8")

36 + manifest = self.write_manifest(root)

37 +

38 + exit_code = publish_safety.main(

39 + [

40 + "backup-existing",

41 + "--root",

42 + str(root),

43 + "--week",

44 + "2026-W23",

45 + "--run-id",

46 + "99",

47 + "--kind",

48 + "content",

49 + "--manifest",

50 + "data/candidates/2026-W23/99/publish-manifest.json",

51 + "--expected-publish-ref",

52 + "abc",

53 + "--actual-publish-ref",

54 + "abc",

55 + "--path",

56 + "content/weekly/2026/W23.md",

57 + ]

58 + )

59 +

60 + self.assertEqual(exit_code, 0)

61 + backup_manifest = root / "data/backups/2026-W23/99/content/manifest.json"

62 + payload = json.loads(backup_manifest.read_text(encoding="utf-8"))

63 + self.assertEqual(payload["schema_version"], "publish_backup_v1")

64 + self.assertEqual(payload["publish_ref"]["expected"], "abc")

65 + self.assertEqual(payload["source_manifest"]["source_artifacts"][0]["sha256"], "raw-sha")

66 + self.assertRegex(payload["files"][0]["sha256"], r"^[0-9a-f]{64}$")

67 +

68 + with self.assertRaises(SystemExit):

69 + publish_safety.main(

70 + [

71 + "backup-existing",

72 + "--root",

73 + str(root),

74 + "--week",

75 + "2026-W23",

76 + "--run-id",

77 + "99",

78 + "--kind",

79 + "content",

80 + "--manifest",

81 + "data/candidates/2026-W23/99/publish-manifest.json",

82 + "--path",

83 + "content/weekly/2026/W23.md",

84 + ]

85 + )

86 +

87 + target.write_text("bad replacement\n", encoding="utf-8")

88 + self.assertEqual(

89 + publish_safety.main(["restore-backup", "--root", str(root), "--backup-manifest", str(backup_manifest)]),

90 + 0,

91 + )

92 + self.assertEqual(target.read_text(encoding="utf-8"), "known good article\n")

93 +

94 + def test_backup_existing_requires_loadable_publish_manifest(self) -> None:

95 + tests_root = Path(__file__).resolve().parent

96 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

97 + root = Path(tmpdir)

98 + target = root / "content/weekly/2026/W23.md"

99 + target.parent.mkdir(parents=True, exist_ok=True)

100 + target.write_text("known good article\n", encoding="utf-8")

101 +

102 + with self.assertRaises(SystemExit):

103 + publish_safety.main(

104 + [

105 + "backup-existing",

106 + "--root",

107 + str(root),

108 + "--week",

109 + "2026-W23",

110 + "--run-id",

111 + "99",

112 + "--kind",

113 + "content",

114 + "--manifest",

115 + "data/candidates/2026-W23/99/publish-manifest.json",

116 + "--path",

117 + "content/weekly/2026/W23.md",

118 + ]

119 + )

120 + self.assertFalse((root / "data/backups").exists())

121 +

122 + malformed_manifest = root / "data/candidates/2026-W23/99/publish-manifest.json"

123 + malformed_manifest.parent.mkdir(parents=True, exist_ok=True)

124 + malformed_manifest.write_text("{not json", encoding="utf-8")

125 +

126 + with self.assertRaises(SystemExit):

127 + publish_safety.main(

128 + [

129 + "backup-existing",

130 + "--root",

131 + str(root),

132 + "--week",

133 + "2026-W23",

134 + "--run-id",

135 + "99",

136 + "--kind",

137 + "content",

138 + "--manifest",

139 + "data/candidates/2026-W23/99/publish-manifest.json",

140 + "--path",

141 + "content/weekly/2026/W23.md",

142 + ]

143 + )

144 + self.assertFalse((root / "data/backups").exists())

145 +

146 + def test_backup_existing_rejects_non_file_targets(self) -> None:

147 + tests_root = Path(__file__).resolve().parent

148 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

149 + root = Path(tmpdir)

150 + self.write_manifest(root)

151 + target_dir = root / "content/weekly/2026"

152 + target_dir.mkdir(parents=True, exist_ok=True)

153 +

154 + with self.assertRaises(SystemExit):

155 + publish_safety.main(

156 + [

157 + "backup-existing",

158 + "--root",

159 + str(root),

160 + "--week",

161 + "2026-W23",

162 + "--run-id",

163 + "99",

164 + "--kind",

165 + "content",

166 + "--manifest",

167 + "data/candidates/2026-W23/99/publish-manifest.json",

168 + "--path",

169 + "content/weekly/2026",

170 + ]

171 + )

172 + self.assertFalse((root / "data/backups").exists())

173 +

174 + def test_restore_backup_rejects_manifest_outside_root(self) -> None:

175 + tests_root = Path(__file__).resolve().parent

176 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

177 + root = Path(tmpdir)

178 + outside_manifest = root.parent / f"{root.name}-outside-manifest.json"

179 + outside_manifest.write_text('{"schema_version":"publish_backup_v1","files":[]}\n', encoding="utf-8")

180 + try:

181 + with self.assertRaises(SystemExit):

182 + publish_safety.main(

183 + [

184 + "restore-backup",

185 + "--root",

186 + str(root),

187 + "--backup-manifest",

188 + str(outside_manifest),

189 + ]

190 + )

191 + with self.assertRaises(SystemExit):

192 + publish_safety.main(

193 + [

194 + "restore-backup",

195 + "--root",

196 + str(root),

197 + "--backup-manifest",

198 + "../outside-manifest.json",

199 + ]

200 + )

201 + finally:

202 + outside_manifest.unlink(missing_ok=True)

203 +

204 +

205 +if __name__ == "__main__":

206 + unittest.main()

tests/test_sync_publish_workflow.py

+21

@@ -2,6 +2,7 @@ from pathlib import Path

4 WORKFLOW = Path(".github/workflows/sync-publish-to-main.yml")

5 +RESTORE_WORKFLOW = Path(".github/workflows/restore-publish-backup.yml")

8 def test_publish_sync_only_checks_out_generated_content_paths() -> None:

@@ -30,3 +31,23 @@ def test_publish_sync_refuses_staged_squad_changes() -> None:

31 assert "git diff --cached --name-only | grep -E '^\\.squad/'" in workflow

32 assert "data/raw/" in workflow

33 assert ".squad/**" in workflow

34 +

35 +

36 +def test_restore_publish_backup_workflow_uses_immutable_backup_manifest() -> None:

37 + workflow = RESTORE_WORKFLOW.read_text(encoding="utf-8")

38 +

39 + assert "backup_manifest" in workflow

40 + assert "python3 ../workflow-source/scripts/publish_safety.py restore-backup" in workflow

41 + assert "--force-with-lease" in workflow

42 + assert "ref: publish" in workflow

43 +

44 +

45 +def test_restore_publish_backup_workflow_keeps_helper_available_after_publish_checkout() -> None:

46 + workflow = RESTORE_WORKFLOW.read_text(encoding="utf-8")

47 +

48 + assert "path: workflow-source" in workflow

49 + assert "ref: ${{ github.sha }}" in workflow

50 + assert "path: publish" in workflow

51 + assert "cd publish" in workflow

52 + assert "python3 scripts/publish_safety.py restore-backup" not in workflow

53 + assert "python3 ../workflow-source/scripts/publish_safety.py restore-backup" in workflow