| 1 | #!/usr/bin/env python3 |
| 2 | from __future__ import annotations |
| 3 | |
| 4 | import argparse |
| 5 | import hashlib |
| 6 | import json |
| 7 | import re |
| 8 | import shutil |
| 9 | from datetime import UTC, datetime |
| 10 | from pathlib import Path |
| 11 | from typing import Any |
| 12 | |
| 13 | BACKUP_SCHEMA_VERSION = "publish_backup_v1" |
| 14 | PUBLISH_MANIFEST_SCHEMA_VERSION = "publish_eligibility_v1" |
| 15 | RAW_STORE_SCHEMA_VERSION = "raw_store_v1" |
| 16 | WEEK_PATTERN = re.compile(r"^[0-9]{4}-W[0-9]{2}$") |
| 17 | SAFE_COMPONENT_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$") |
| 18 | |
| 19 | |
| 20 | def sha256_file(path: Path) -> str | None: |
| 21 | if not path.exists() or not path.is_file(): |
| 22 | return None |
| 23 | digest = hashlib.sha256() |
| 24 | with path.open("rb") as handle: |
| 25 | for chunk in iter(lambda: handle.read(1024 * 1024), b""): |
| 26 | digest.update(chunk) |
| 27 | return digest.hexdigest() |
| 28 | |
| 29 | |
| 30 | def load_json(path: Path) -> dict[str, Any] | None: |
| 31 | try: |
| 32 | payload = json.loads(path.read_text(encoding="utf-8")) |
| 33 | except (FileNotFoundError, OSError, json.JSONDecodeError): |
| 34 | return None |
| 35 | return payload if isinstance(payload, dict) else None |
| 36 | |
| 37 | |
| 38 | def relpath_under_root(root: Path, value: str) -> Path: |
| 39 | path = Path(value) |
| 40 | if path.is_absolute(): |
| 41 | raise SystemExit(f"Path must be relative to repository root: {value}") |
| 42 | resolved_root = root.resolve() |
| 43 | resolved_path = (resolved_root / path).resolve() |
| 44 | try: |
| 45 | return resolved_path.relative_to(resolved_root) |
| 46 | except ValueError as exc: |
| 47 | raise SystemExit(f"Path must stay under repository root: {value}") from exc |
| 48 | |
| 49 | |
| 50 | def path_under_root(root: Path, value: Path) -> tuple[Path, Path]: |
| 51 | resolved_root = root.resolve() |
| 52 | resolved_path = (value if value.is_absolute() else resolved_root / value).resolve() |
| 53 | try: |
| 54 | relative = resolved_path.relative_to(resolved_root) |
| 55 | except ValueError as exc: |
| 56 | raise SystemExit(f"Path must stay under repository root: {value}") from exc |
| 57 | return relative, resolved_path |
| 58 | |
| 59 | |
| 60 | def require_safe_component(value: str, *, label: str) -> str: |
| 61 | candidate = value.strip() |
| 62 | if not candidate or not SAFE_COMPONENT_PATTERN.fullmatch(candidate): |
| 63 | raise SystemExit(f"Invalid {label}: {value!r}") |
| 64 | return candidate |
| 65 | |
| 66 | |
| 67 | def require_week(value: str) -> str: |
| 68 | week = value.strip() |
| 69 | if not WEEK_PATTERN.fullmatch(week): |
| 70 | raise SystemExit(f"Invalid week format. Expected YYYY-WNN, got: {value!r}") |
| 71 | return week |
| 72 | |
| 73 | |
| 74 | def require_raw_path(root: Path, value: str) -> tuple[Path, Path]: |
| 75 | relative = relpath_under_root(root, value) |
| 76 | if relative.parts[:2] != ("data", "raw"): |
| 77 | raise SystemExit(f"Raw store paths must live under data/raw/: {value}") |
| 78 | return relative, root / relative |
| 79 | |
| 80 | |
| 81 | def is_week_raw_json(path: Path, week: str) -> bool: |
| 82 | return path.suffix == ".json" and ( |
| 83 | path.name == f"{week}.json" or path.name.startswith(f"{week}-") |
| 84 | ) |
| 85 | |
| 86 | |
| 87 | def parse_args(argv: list[str] | None = None) -> argparse.Namespace: |
| 88 | parser = argparse.ArgumentParser(description="Publish-branch backup and restore safeguards.") |
| 89 | subparsers = parser.add_subparsers(dest="command", required=True) |
| 90 | |
| 91 | backup = subparsers.add_parser( |
| 92 | "backup-existing", help="Create an immutable backup manifest for target paths." |
| 93 | ) |
| 94 | backup.add_argument("--root", default=".", type=Path) |
| 95 | backup.add_argument("--week", required=True) |
| 96 | backup.add_argument("--run-id", required=True) |
| 97 | backup.add_argument("--kind", required=True, choices=["analysis", "content"]) |
| 98 | backup.add_argument( |
| 99 | "--manifest", required=True, type=Path, help="Publish eligibility manifest." |
| 100 | ) |
| 101 | backup.add_argument("--expected-publish-ref", default="") |
| 102 | backup.add_argument("--actual-publish-ref", default="") |
| 103 | backup.add_argument("--backup-root", default=Path("data/backups"), type=Path) |
| 104 | backup.add_argument( |
| 105 | "--path", |
| 106 | action="append", |
| 107 | required=True, |
| 108 | help="Published path to snapshot before replacement.", |
| 109 | ) |
| 110 | |
| 111 | restore = subparsers.add_parser( |
| 112 | "restore-backup", help="Restore files from an immutable publish backup manifest." |
| 113 | ) |
| 114 | restore.add_argument("--root", default=".", type=Path) |
| 115 | restore.add_argument("--backup-manifest", required=True, type=Path) |
| 116 | |
| 117 | store_raw_parser = subparsers.add_parser( |
| 118 | "store-raw", help="Store raw evidence under an immutable week/source-run path." |
| 119 | ) |
| 120 | store_raw_parser.add_argument("--root", default=".", type=Path) |
| 121 | store_raw_parser.add_argument("--week", required=True) |
| 122 | store_raw_parser.add_argument("--source-run-id", required=True) |
| 123 | store_raw_parser.add_argument("--source-artifact-id", required=True) |
| 124 | store_raw_parser.add_argument("--source-artifact-name", default="raw-data") |
| 125 | store_raw_parser.add_argument("--source-head-sha", required=True) |
| 126 | store_raw_parser.add_argument("--store-root", default=Path("data/raw-store"), type=Path) |
| 127 | store_raw_parser.add_argument( |
| 128 | "--path", |
| 129 | action="append", |
| 130 | required=True, |
| 131 | help="Week-scoped data/raw file to preserve.", |
| 132 | ) |
| 133 | |
| 134 | restore_raw_parser = subparsers.add_parser( |
| 135 | "restore-raw", help="Restore hash-verified raw evidence from a source workflow run." |
| 136 | ) |
| 137 | restore_raw_parser.add_argument("--root", default=".", type=Path) |
| 138 | restore_raw_parser.add_argument("--week", required=True) |
| 139 | restore_raw_parser.add_argument("--source-run-id", required=True) |
| 140 | restore_raw_parser.add_argument("--store-root", default=Path("data/raw-store"), type=Path) |
| 141 | |
| 142 | return parser.parse_args(argv) |
| 143 | |
| 144 | |
| 145 | def backup_existing(args: argparse.Namespace) -> int: |
| 146 | root = args.root.resolve() |
| 147 | source_manifest_relative = relpath_under_root(root, args.manifest.as_posix()) |
| 148 | source_manifest = root / source_manifest_relative |
| 149 | source_manifest_payload = load_json(source_manifest) |
| 150 | if source_manifest_payload is None: |
| 151 | raise SystemExit( |
| 152 | f"Publish manifest is missing or malformed: {source_manifest_relative.as_posix()}" |
| 153 | ) |
| 154 | if source_manifest_payload.get("schema_version") != PUBLISH_MANIFEST_SCHEMA_VERSION: |
| 155 | raise SystemExit( |
| 156 | f"Unsupported publish manifest schema: {source_manifest_payload.get('schema_version')!r}" |
| 157 | ) |
| 158 | if not isinstance(source_manifest_payload.get("candidate"), dict): |
| 159 | raise SystemExit("Publish manifest lacks candidate block.") |
| 160 | if not isinstance(source_manifest_payload.get("source_artifacts"), list): |
| 161 | raise SystemExit("Publish manifest lacks source artifact provenance.") |
| 162 | if not isinstance(source_manifest_payload.get("analysis"), dict): |
| 163 | raise SystemExit("Publish manifest lacks analysis provenance.") |
| 164 | |
| 165 | entries: list[dict[str, Any]] = [] |
| 166 | for raw_path in args.path: |
| 167 | relative = relpath_under_root(root, raw_path) |
| 168 | source = root / relative |
| 169 | if source.exists() and not source.is_file(): |
| 170 | raise SystemExit(f"Backup target must be a regular file: {relative.as_posix()}") |
| 171 | entries.append( |
| 172 | { |
| 173 | "path": relative.as_posix(), |
| 174 | "existed": source.exists(), |
| 175 | "size_bytes": source.stat().st_size if source.exists() else 0, |
| 176 | "sha256": sha256_file(source), |
| 177 | } |
| 178 | ) |
| 179 | |
| 180 | backup_root = root / relpath_under_root(root, args.backup_root.as_posix()) |
| 181 | backup_dir = backup_root / args.week / args.run_id / args.kind |
| 182 | manifest_path = backup_dir / "manifest.json" |
| 183 | if manifest_path.exists(): |
| 184 | raise SystemExit(f"Refusing to overwrite immutable backup manifest: {manifest_path}") |
| 185 | backup_dir.mkdir(parents=True, exist_ok=False) |
| 186 | |
| 187 | files_dir = backup_dir / "files" |
| 188 | for entry in entries: |
| 189 | relative = Path(entry["path"]) |
| 190 | source = root / relative |
| 191 | if source.exists(): |
| 192 | destination = files_dir / relative |
| 193 | destination.parent.mkdir(parents=True, exist_ok=True) |
| 194 | shutil.copy2(source, destination) |
| 195 | entry["backup_path"] = destination.relative_to(root).as_posix() |
| 196 | |
| 197 | manifest = { |
| 198 | "schema_version": BACKUP_SCHEMA_VERSION, |
| 199 | "created_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), |
| 200 | "week": args.week, |
| 201 | "run_id": args.run_id, |
| 202 | "kind": args.kind, |
| 203 | "publish_ref": { |
| 204 | "expected": args.expected_publish_ref, |
| 205 | "actual": args.actual_publish_ref, |
| 206 | }, |
| 207 | "source_manifest": { |
| 208 | "path": source_manifest_relative.as_posix(), |
| 209 | "sha256": sha256_file(source_manifest), |
| 210 | "candidate": source_manifest_payload.get("candidate") |
| 211 | if source_manifest_payload |
| 212 | else None, |
| 213 | "source_artifacts": source_manifest_payload.get("source_artifacts") |
| 214 | if source_manifest_payload |
| 215 | else None, |
| 216 | "analysis": source_manifest_payload.get("analysis") |
| 217 | if source_manifest_payload |
| 218 | else None, |
| 219 | }, |
| 220 | "files": entries, |
| 221 | } |
| 222 | manifest_path.write_text( |
| 223 | json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" |
| 224 | ) |
| 225 | print(f"Created immutable publish backup: {manifest_path.relative_to(root).as_posix()}") |
| 226 | return 0 |
| 227 | |
| 228 | |
| 229 | def restore_backup(args: argparse.Namespace) -> int: |
| 230 | root = args.root.resolve() |
| 231 | backup_manifest_relative, backup_manifest = path_under_root(root, args.backup_manifest) |
| 232 | payload = load_json(backup_manifest) |
| 233 | if payload is None: |
| 234 | raise SystemExit(f"Backup manifest is missing or malformed: {backup_manifest}") |
| 235 | if payload.get("schema_version") != BACKUP_SCHEMA_VERSION: |
| 236 | raise SystemExit(f"Unsupported backup schema: {payload.get('schema_version')!r}") |
| 237 | files = payload.get("files") |
| 238 | if not isinstance(files, list): |
| 239 | raise SystemExit("Backup manifest has no files list.") |
| 240 | |
| 241 | for entry in files: |
| 242 | if not isinstance(entry, dict) or not isinstance(entry.get("path"), str): |
| 243 | raise SystemExit("Backup file entry is malformed.") |
| 244 | target_relative = relpath_under_root(root, entry["path"]) |
| 245 | target = root / target_relative |
| 246 | if entry.get("existed") is True: |
| 247 | backup_path = entry.get("backup_path") |
| 248 | if not isinstance(backup_path, str): |
| 249 | raise SystemExit(f"Backup entry lacks backup_path for {entry['path']}") |
| 250 | source_relative = relpath_under_root(root, backup_path) |
| 251 | source = root / source_relative |
| 252 | if sha256_file(source) != entry.get("sha256"): |
| 253 | raise SystemExit(f"Backup checksum mismatch for {backup_path}") |
| 254 | target.parent.mkdir(parents=True, exist_ok=True) |
| 255 | shutil.copy2(source, target) |
| 256 | else: |
| 257 | target.unlink(missing_ok=True) |
| 258 | print(f"Restored publish backup: {backup_manifest_relative.as_posix()}") |
| 259 | return 0 |
| 260 | |
| 261 | |
| 262 | def store_raw(args: argparse.Namespace) -> int: |
| 263 | root = args.root.resolve() |
| 264 | week = require_week(args.week) |
| 265 | source_run_id = require_safe_component(args.source_run_id, label="source_run_id") |
| 266 | source_artifact_id = args.source_artifact_id.strip() |
| 267 | source_artifact_name = args.source_artifact_name.strip() |
| 268 | source_head_sha = args.source_head_sha.strip() |
| 269 | if not source_artifact_id: |
| 270 | raise SystemExit("source_artifact_id is required.") |
| 271 | if not source_artifact_name: |
| 272 | raise SystemExit("source_artifact_name is required.") |
| 273 | if not source_head_sha: |
| 274 | raise SystemExit("source_head_sha is required.") |
| 275 | |
| 276 | sources: list[tuple[Path, Path]] = [] |
| 277 | seen_sources: set[Path] = set() |
| 278 | for raw_path in args.path: |
| 279 | relative, source = require_raw_path(root, raw_path) |
| 280 | if relative in seen_sources: |
| 281 | raise SystemExit(f"Duplicate raw store source: {relative.as_posix()}") |
| 282 | seen_sources.add(relative) |
| 283 | if not source.exists() or not source.is_file(): |
| 284 | raise SystemExit(f"Raw store source must be a regular file: {relative.as_posix()}") |
| 285 | if relative.name != f"{week}.json" and not relative.name.startswith(f"{week}-"): |
| 286 | raise SystemExit(f"Raw store source does not belong to {week}: {relative.as_posix()}") |
| 287 | sources.append((relative, source)) |
| 288 | |
| 289 | store_root = root / relpath_under_root(root, args.store_root.as_posix()) |
| 290 | destination = store_root / week / source_run_id |
| 291 | if destination.exists(): |
| 292 | raise SystemExit(f"Refusing to overwrite immutable raw store: {destination}") |
| 293 | destination.mkdir(parents=True, exist_ok=False) |
| 294 | |
| 295 | entries: list[dict[str, Any]] = [] |
| 296 | try: |
| 297 | for relative, source in sorted(sources): |
| 298 | stored = destination / "files" / relative |
| 299 | stored.parent.mkdir(parents=True, exist_ok=True) |
| 300 | shutil.copy2(source, stored) |
| 301 | source_hash = sha256_file(source) |
| 302 | stored_hash = sha256_file(stored) |
| 303 | if source_hash is None or stored_hash != source_hash: |
| 304 | raise SystemExit(f"Raw store checksum mismatch while copying {relative.as_posix()}") |
| 305 | entries.append( |
| 306 | { |
| 307 | "week": week, |
| 308 | "artifact_id": source_artifact_id, |
| 309 | "source_run_id": source_run_id, |
| 310 | "head_sha": source_head_sha, |
| 311 | "original_path": relative.as_posix(), |
| 312 | "stored_path": stored.relative_to(root).as_posix(), |
| 313 | "size_bytes": stored.stat().st_size, |
| 314 | "sha256": stored_hash, |
| 315 | } |
| 316 | ) |
| 317 | |
| 318 | manifest = { |
| 319 | "schema_version": RAW_STORE_SCHEMA_VERSION, |
| 320 | "created_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), |
| 321 | "week": week, |
| 322 | "source_run_id": source_run_id, |
| 323 | "source_artifact": { |
| 324 | "id": source_artifact_id, |
| 325 | "name": source_artifact_name, |
| 326 | "head_sha": source_head_sha, |
| 327 | "retention_days": 90, |
| 328 | }, |
| 329 | "files": entries, |
| 330 | } |
| 331 | manifest_path = destination / "manifest.json" |
| 332 | manifest_path.write_text( |
| 333 | json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" |
| 334 | ) |
| 335 | except BaseException: |
| 336 | shutil.rmtree(destination, ignore_errors=True) |
| 337 | raise |
| 338 | |
| 339 | print(f"Created immutable raw store: {destination.relative_to(root).as_posix()}") |
| 340 | return 0 |
| 341 | |
| 342 | |
| 343 | def validate_raw_store_manifest( |
| 344 | root: Path, |
| 345 | manifest_path: Path, |
| 346 | *, |
| 347 | expected_week: str, |
| 348 | expected_source_run_id: str, |
| 349 | ) -> tuple[dict[str, Any], list[tuple[Path, Path, dict[str, Any]]]]: |
| 350 | root = root.resolve() |
| 351 | week = require_week(expected_week) |
| 352 | source_run_id = require_safe_component(expected_source_run_id, label="source_run_id") |
| 353 | manifest_relative, manifest = path_under_root(root, manifest_path) |
| 354 | payload = load_json(manifest) |
| 355 | if payload is None: |
| 356 | raise SystemExit(f"Raw store manifest is missing or malformed: {manifest_relative}") |
| 357 | if payload.get("schema_version") != RAW_STORE_SCHEMA_VERSION: |
| 358 | raise SystemExit(f"Unsupported raw store schema: {payload.get('schema_version')!r}") |
| 359 | if payload.get("week") != week: |
| 360 | raise SystemExit(f"Raw store week mismatch: expected {week}, found {payload.get('week')!r}") |
| 361 | if payload.get("source_run_id") != source_run_id: |
| 362 | raise SystemExit( |
| 363 | "Raw store source_run_id mismatch: " |
| 364 | f"expected {source_run_id}, found {payload.get('source_run_id')!r}" |
| 365 | ) |
| 366 | source_artifact = payload.get("source_artifact") |
| 367 | artifact_name = source_artifact.get("name") if isinstance(source_artifact, dict) else None |
| 368 | if ( |
| 369 | not isinstance(source_artifact, dict) |
| 370 | or not source_artifact.get("id") |
| 371 | or not (isinstance(artifact_name, str) and artifact_name.strip()) |
| 372 | or not source_artifact.get("head_sha") |
| 373 | ): |
| 374 | raise SystemExit("Raw store manifest lacks source artifact provenance.") |
| 375 | files = payload.get("files") |
| 376 | if not isinstance(files, list) or not files: |
| 377 | raise SystemExit("Raw store manifest has no files list.") |
| 378 | |
| 379 | verified: list[tuple[Path, Path, dict[str, Any]]] = [] |
| 380 | store_directory = manifest.parent.resolve() |
| 381 | store_directory_relative = manifest.parent.relative_to(root) |
| 382 | seen_original_paths: set[Path] = set() |
| 383 | for entry in files: |
| 384 | if not isinstance(entry, dict): |
| 385 | raise SystemExit("Raw store file entry is malformed.") |
| 386 | original_path = entry.get("original_path") |
| 387 | stored_path = entry.get("stored_path") |
| 388 | if not isinstance(original_path, str) or not isinstance(stored_path, str): |
| 389 | raise SystemExit("Raw store file entry lacks original_path or stored_path.") |
| 390 | if entry.get("week") != week or entry.get("source_run_id") != source_run_id: |
| 391 | raise SystemExit(f"Raw store file provenance mismatch for {original_path}") |
| 392 | if entry.get("artifact_id") != source_artifact.get("id") or entry.get( |
| 393 | "head_sha" |
| 394 | ) != source_artifact.get("head_sha"): |
| 395 | raise SystemExit(f"Raw store artifact provenance mismatch for {original_path}") |
| 396 | |
| 397 | target_relative, target = require_raw_path(root, original_path) |
| 398 | if target_relative in seen_original_paths: |
| 399 | raise SystemExit(f"Duplicate raw store original_path: {original_path}") |
| 400 | seen_original_paths.add(target_relative) |
| 401 | stored_relative = relpath_under_root(root, stored_path) |
| 402 | expected_stored_relative = store_directory_relative / "files" / target_relative |
| 403 | if stored_relative != expected_stored_relative: |
| 404 | raise SystemExit(f"Raw store stored_path mismatch for {original_path}: {stored_path}") |
| 405 | stored = root / stored_relative |
| 406 | try: |
| 407 | stored.resolve().relative_to(store_directory) |
| 408 | except ValueError as exc: |
| 409 | raise SystemExit( |
| 410 | f"Raw store file must stay under its immutable run directory: {stored_path}" |
| 411 | ) from exc |
| 412 | if not stored.exists() or not stored.is_file(): |
| 413 | raise SystemExit(f"Raw store file is missing: {stored_relative.as_posix()}") |
| 414 | if stored.stat().st_size != entry.get("size_bytes"): |
| 415 | raise SystemExit(f"Raw store size mismatch for {stored_relative.as_posix()}") |
| 416 | if sha256_file(stored) != entry.get("sha256"): |
| 417 | raise SystemExit(f"Raw store checksum mismatch for {stored_relative.as_posix()}") |
| 418 | verified.append((stored, target, entry)) |
| 419 | |
| 420 | required_raw_path = Path("data") / "raw" / f"{week}.json" |
| 421 | if required_raw_path not in seen_original_paths: |
| 422 | raise SystemExit(f"Raw store manifest lacks required payload: {required_raw_path}") |
| 423 | return payload, verified |
| 424 | |
| 425 | |
| 426 | def restore_raw(args: argparse.Namespace) -> int: |
| 427 | root = args.root.resolve() |
| 428 | week = require_week(args.week) |
| 429 | source_run_id = require_safe_component(args.source_run_id, label="source_run_id") |
| 430 | store_root = root / relpath_under_root(root, args.store_root.as_posix()) |
| 431 | manifest = store_root / week / source_run_id / "manifest.json" |
| 432 | _, verified = validate_raw_store_manifest( |
| 433 | root, |
| 434 | manifest, |
| 435 | expected_week=week, |
| 436 | expected_source_run_id=source_run_id, |
| 437 | ) |
| 438 | |
| 439 | expected_targets = {target for _, target, _ in verified} |
| 440 | raw_root = root / "data" / "raw" |
| 441 | if raw_root.exists(): |
| 442 | for existing in sorted(raw_root.rglob("*.json")): |
| 443 | if ( |
| 444 | existing.is_file() |
| 445 | and is_week_raw_json(existing, week) |
| 446 | and existing not in expected_targets |
| 447 | ): |
| 448 | existing.unlink() |
| 449 | |
| 450 | for stored, target, _ in verified: |
| 451 | target.parent.mkdir(parents=True, exist_ok=True) |
| 452 | shutil.copy2(stored, target) |
| 453 | print( |
| 454 | "Restored hash-verified raw evidence: " |
| 455 | f"week={week} source_run_id={source_run_id} files={len(verified)}" |
| 456 | ) |
| 457 | return 0 |
| 458 | |
| 459 | |
| 460 | def main(argv: list[str] | None = None) -> int: |
| 461 | args = parse_args(argv) |
| 462 | if args.command == "backup-existing": |
| 463 | return backup_existing(args) |
| 464 | if args.command == "restore-backup": |
| 465 | return restore_backup(args) |
| 466 | if args.command == "store-raw": |
| 467 | return store_raw(args) |
| 468 | if args.command == "restore-raw": |
| 469 | return restore_raw(args) |
| 470 | raise AssertionError(args.command) |
| 471 | |
| 472 | |
| 473 | if __name__ == "__main__": |
| 474 | raise SystemExit(main()) |