Block no-AI fallback from replacing AI-authored summaries (#277)
* fix: block no-ai fallback promotion (#251) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: lower no-ai fallback quality score Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
Jun 6, 2026 at 21:50 UTC
be0c2d5e64c4ba47fcf3fdf955dab215ac72c9eb
6 files changed
+470
-9
scripts/analyze_fallback.py
+2
-1
@@ -25,6 +25,7 @@ DEFAULT_SKILLS_DIR = ROOT / ".squad" / "skills"
25
DEFAULT_MODELS_ENDPOINT = "https://models.github.ai/inference/chat/completions"
26
DEFAULT_MODELS_MODEL = "openai/gpt-4o"
27
DEFAULT_MODELS_TIMEOUT = 30
28
+NO_AI_DIAGNOSTIC_QUALITY_SCORE = 40
29
30
31
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
@@ -470,7 +471,7 @@ categories: [weekly]
471
repos_featured: {repos_featured}
472
stars_tracked: {total_stars}
473
top_repo: "{top_repo}"
473
-quality_score: 62
474
+quality_score: {NO_AI_DIAGNOSTIC_QUALITY_SCORE}
475
summary: "Automated data-only summary for {week}. AI analysis was unavailable; this report presents raw crawl statistics and top repositories without editorial commentary."
476
---
477
scripts/promotion_guard.py
+123
-2
@@ -16,6 +16,8 @@ class PromotionBlocked(ValueError):
16
17
18
WEEK_PATTERN = re.compile(r"^(?P<year>\d{4})-W(?P<week>\d{2})$")
19
+FRONTMATTER_PATTERN = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL)
20
+FALLBACK_MIN_QUALITY_SCORE = 70
21
22
23
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
@@ -71,6 +73,59 @@ def _resolve_under_root(root: Path, value: Any, field: str, reasons: list[str])
73
return resolved_path
74
75
76
+def _frontmatter(path: Path) -> dict[str, Any]:
77
+ if not path.exists() or not path.is_file():
78
+ return {}
79
+ match = FRONTMATTER_PATTERN.match(path.read_text(encoding="utf-8"))
80
+ if not match:
81
+ return {}
82
+ result: dict[str, Any] = {}
83
+ for line in match.group(1).splitlines():
84
+ if ":" not in line or line.startswith((" ", "\t")):
85
+ continue
86
+ key, value = line.split(":", 1)
87
+ scalar = value.strip().strip('"').strip("'")
88
+ result[key.strip()] = int(scalar) if scalar.isdigit() else scalar
89
+ return result
90
+
91
+
92
+def _is_no_ai_summary(path: Path) -> bool:
93
+ if not path.exists():
94
+ return False
95
+ text = path.read_text(encoding="utf-8").lower()
96
+ return any(
97
+ marker in text
98
+ for marker in (
99
+ "source: no-ai",
100
+ "model: none",
101
+ "without ai-powered analysis",
102
+ "generated without ai assistance",
103
+ "automated data-only summary",
104
+ )
105
+ )
106
+
107
+
108
+def _has_existing_good_ai_article(root: Path, week: str) -> bool:
109
+ path = root / "data" / "analyzed" / f"{week}-summary.md"
110
+ score = _frontmatter(path).get("quality_score")
111
+ return path.exists() and isinstance(score, int) and score >= 60 and not _is_no_ai_summary(path)
112
+
113
+
114
+def _promotion_policy(manifest: dict[str, Any]) -> dict[str, Any]:
115
+ policy = manifest.get("promotion_policy")
116
+ if isinstance(policy, dict):
117
+ return policy
118
+ promotion = manifest.get("promotion")
119
+ if isinstance(promotion, dict):
120
+ audit = manifest.get("audit") if isinstance(manifest.get("audit"), dict) else {}
121
+ return {
122
+ "mode": promotion.get("policy", "default"),
123
+ "reason": promotion.get("reason") or audit.get("reason"),
124
+ "actor": audit.get("actor"),
125
+ }
126
+ return {"mode": "default"}
127
+
128
+
129
def _validate_manifest(manifest: dict[str, Any], root: Path, manifest_path: Path) -> tuple[str, Path, Path, list[str]]:
130
reasons: list[str] = []
131
@@ -84,18 +139,57 @@ def _validate_manifest(manifest: dict[str, Any], root: Path, manifest_path: Path
139
elif not WEEK_PATTERN.fullmatch(week):
140
reasons.append("week must use YYYY-WNN format.")
141
87
- if manifest.get("promotion_eligible") is not True:
142
+ promotion_eligible = manifest.get("promotion_eligible")
143
+ if promotion_eligible is None and isinstance(manifest.get("promotion"), dict):
144
+ promotion_eligible = manifest["promotion"].get("eligible")
145
+ if promotion_eligible is not True:
146
reasons.append("promotion_eligible must be true.")
147
148
candidate_summary = _resolve_under_root(root, manifest.get("candidate_summary_path"), "candidate_summary_path", reasons)
149
candidate_content = _resolve_under_root(root, manifest.get("candidate_content_path"), "candidate_content_path", reasons)
150
151
+ policy = _promotion_policy(manifest)
152
+ policy_mode = str(policy.get("mode") or "default")
153
ai_provenance = manifest.get("ai_provenance")
154
+ if not isinstance(ai_provenance, dict) and isinstance(manifest.get("analysis"), dict):
155
+ analysis = manifest["analysis"]
156
+ provenance = analysis.get("provenance") if isinstance(analysis.get("provenance"), dict) else {}
157
+ ai_provenance = {
158
+ "source": analysis.get("source"),
159
+ "model": analysis.get("model"),
160
+ "degraded": analysis.get("ai_status") not in {"ai", "no-ai"},
161
+ "authorship": provenance.get("authorship"),
162
+ "fallback_reason": provenance.get("fallback_reason"),
163
+ "attempted_ai_paths": provenance.get("attempted_ai_paths"),
164
+ }
165
if not isinstance(ai_provenance, dict):
166
reasons.append("ai_provenance is required.")
167
else:
168
source = ai_provenance.get("source")
98
- if source in {None, "", "no-ai"}:
169
+ is_no_ai = source == "no-ai" or ai_provenance.get("authorship") == "no-ai-fallback"
170
+ if is_no_ai:
171
+ existing_good_ai = _has_existing_good_ai_article(root, str(week))
172
+ if policy_mode == "force-replace":
173
+ if not policy.get("reason"):
174
+ reasons.append("force-replace requires a reason.")
175
+ if not policy.get("actor"):
176
+ reasons.append("force-replace requires an actor.")
177
+ elif policy_mode == "allow-no-ai-first-publish":
178
+ if existing_good_ai:
179
+ reasons.append("no-AI fallback cannot first-publish over an existing good AI-authored article.")
180
+ else:
181
+ reasons.append("no-AI fallback is ineligible for default promotion.")
182
+ if existing_good_ai:
183
+ reasons.append("no-AI fallback is ineligible to replace an existing good AI-authored article by default.")
184
+ if candidate_summary is not None:
185
+ quality_score = _frontmatter(candidate_summary).get("quality_score")
186
+ if not isinstance(quality_score, int) or quality_score < FALLBACK_MIN_QUALITY_SCORE:
187
+ reasons.append(f"no-AI fallback quality_score must be at least {FALLBACK_MIN_QUALITY_SCORE}.")
188
+ if not ai_provenance.get("fallback_reason"):
189
+ reasons.append("no-AI fallback provenance requires fallback_reason.")
190
+ if not ai_provenance.get("attempted_ai_paths"):
191
+ reasons.append("no-AI fallback provenance requires attempted_ai_paths.")
192
+ elif source in {None, ""}:
193
reasons.append("AI-authored provenance is required for normal promotion.")
194
if ai_provenance.get("degraded") is True:
195
reasons.append("degraded AI provenance is not eligible for normal promotion.")
@@ -160,6 +254,32 @@ def _write_diagnostic(root: Path, week: str, manifest: dict[str, Any] | None, re
254
return diagnostic_path
255
256
257
+def _write_force_audit(root: Path, week: str, manifest: dict[str, Any]) -> Path | None:
258
+ policy = _promotion_policy(manifest)
259
+ if policy.get("mode") != "force-replace":
260
+ return None
261
+ diagnostic_dir = root / "data" / "diagnostics" / "promotion"
262
+ diagnostic_dir.mkdir(parents=True, exist_ok=True)
263
+ audit_path = diagnostic_dir / f"{week}-force-replace-audit.json"
264
+ audit_path.write_text(
265
+ json.dumps(
266
+ {
267
+ "week": week,
268
+ "mode": "force-replace",
269
+ "actor": policy.get("actor"),
270
+ "reason": policy.get("reason"),
271
+ "source_artifacts": manifest.get("source_artifacts", []),
272
+ "manifest": manifest,
273
+ },
274
+ indent=2,
275
+ sort_keys=True,
276
+ )
277
+ + "\n",
278
+ encoding="utf-8",
279
+ )
280
+ return audit_path
281
+
282
+
283
def promote_candidate(manifest_path: Path, *, root: Path | None = None) -> tuple[Path, Path]:
284
workspace = (root or Path.cwd()).resolve()
285
resolved_manifest_path = manifest_path if manifest_path.is_absolute() else workspace / manifest_path
@@ -183,6 +303,7 @@ def promote_candidate(manifest_path: Path, *, root: Path | None = None) -> tuple
303
canonical_content = workspace / "content" / "weekly" / year / f"W{week_number}.md"
304
canonical_summary.parent.mkdir(parents=True, exist_ok=True)
305
canonical_content.parent.mkdir(parents=True, exist_ok=True)
306
+ _write_force_audit(workspace, week, manifest)
307
shutil.copyfile(candidate_summary, canonical_summary)
308
shutil.copyfile(candidate_content, canonical_content)
309
return canonical_summary, canonical_content
scripts/publish_manifest.py
+113
-5
@@ -12,7 +12,9 @@ from typing import Any
12
13
SCHEMA_VERSION = "publish_eligibility_v1"
14
AI_SOURCES = {"copilot-cli", "github-models"}
15
+NO_AI_SOURCE = "no-ai"
16
MIN_PUBLISH_QUALITY_SCORE = 60
17
+FALLBACK_MIN_QUALITY_SCORE = 70
18
NO_AI_MARKERS = (
19
"AI analysis was unavailable",
20
"without AI-powered analysis",
@@ -38,6 +40,25 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
40
create.add_argument("--validation-status", choices=["passed", "failed"], required=True)
41
create.add_argument("--output", required=True, type=Path)
42
create.add_argument("--artifact", action="append", default=[], help="Additional source artifact as role=path.")
43
+ create.add_argument(
44
+ "--fallback-reason",
45
+ default="",
46
+ help="Required reason when analysis-source is no-ai; records why AI output was unavailable.",
47
+ )
48
+ create.add_argument(
49
+ "--attempted-ai-path",
50
+ action="append",
51
+ default=[],
52
+ help="AI path attempted before this candidate, e.g. provider=copilot-cli,model=copilot-default,status=failed.",
53
+ )
54
+ create.add_argument(
55
+ "--publish-policy",
56
+ choices=["default", "allow-no-ai-first-publish", "force-replace"],
57
+ default="default",
58
+ help="Explicit operator policy for no-AI fallback publication.",
59
+ )
60
+ create.add_argument("--force-reason", default="", help="Operator reason required for force-replace.")
61
+ create.add_argument("--actor", default="", help="Operator or automation actor requesting explicit fallback policy.")
62
63
check = subparsers.add_parser("assert-eligible", help="Fail unless the manifest permits promotion.")
64
check.add_argument("--manifest", required=True, type=Path)
@@ -209,6 +230,18 @@ def published_summary_status(path: Path, week: str) -> dict[str, Any]:
230
return status
231
232
233
+def fallback_quality_errors(summary: Path, validation_passed: bool) -> list[str]:
234
+ errors: list[str] = []
235
+ if not validation_passed:
236
+ errors.append("no-AI fallback cannot publish because analysis validation did not pass")
237
+ quality_score = markdown_metadata(summary).get("quality_score")
238
+ if not isinstance(quality_score, (int, float)) or quality_score < FALLBACK_MIN_QUALITY_SCORE:
239
+ errors.append(
240
+ f"no-AI fallback quality_score must be at least {FALLBACK_MIN_QUALITY_SCORE} for explicit fallback publication"
241
+ )
242
+ return errors
243
+
244
+
245
def same_day_reuse_status(payload: dict[str, Any] | None) -> dict[str, Any]:
246
metadata = payload.get("metadata", {}) if isinstance(payload, dict) else {}
247
if not isinstance(metadata, dict):
@@ -306,12 +339,14 @@ def create_manifest(args: argparse.Namespace) -> int:
339
]
340
341
analysis_source = args.analysis_source.strip()
309
- ai_status = "ai" if analysis_source in AI_SOURCES else "no-ai" if analysis_source == "no-ai" else "unknown"
342
+ ai_status = "ai" if analysis_source in AI_SOURCES else "no-ai" if analysis_source == NO_AI_SOURCE else "unknown"
343
candidate_metadata = markdown_metadata(args.summary)
344
published_status = published_summary_status(args.published_summary, args.week)
345
candidate_exists = args.summary.exists()
346
validation_passed = args.validation_status == "passed"
347
candidate_quality = candidate_metadata.get("quality_score")
348
+ attempted_ai_paths = [path for path in args.attempted_ai_path if path.strip()]
349
+ force_replacing_no_ai = ai_status == "no-ai" and args.publish_policy == "force-replace"
350
comparison_reasons: list[str] = []
351
if candidate_exists:
352
if candidate_metadata.get("week") not in {None, args.week}:
@@ -322,25 +357,56 @@ def create_manifest(args: argparse.Namespace) -> int:
357
comparison_reasons.append("candidate summary lacks quality_score")
358
elif candidate_quality < MIN_PUBLISH_QUALITY_SCORE:
359
comparison_reasons.append(f"candidate quality_score below {MIN_PUBLISH_QUALITY_SCORE}: {candidate_quality}")
325
- if published_status.get("good") and isinstance(candidate_quality, (int, float)):
360
+ if published_status.get("good") and isinstance(candidate_quality, (int, float)) and not force_replacing_no_ai:
361
published_quality = published_status.get("quality_score")
362
if isinstance(published_quality, (int, float)) and candidate_quality < published_quality:
363
comparison_reasons.append(
364
f"candidate quality_score {candidate_quality} is lower than published good quality_score {published_quality}"
365
)
366
332
- eligible = candidate_exists and validation_passed and ai_status == "ai" and not artifact_reasons and not comparison_reasons
333
-
367
reasons: list[str] = []
368
+ fallback_errors: list[str] = []
369
+ if ai_status == "no-ai":
370
+ if not args.fallback_reason.strip():
371
+ reasons.append("fallback_reason is required for no-AI fallback candidates")
372
+ if not attempted_ai_paths:
373
+ reasons.append("attempted_ai_paths must record attempted AI paths for no-AI fallback candidates")
374
+ if args.publish_policy == "default":
375
+ if published_status.get("good"):
376
+ reasons.append("no-AI fallback is ineligible to replace an existing good AI-authored article by default")
377
+ else:
378
+ reasons.append("no-AI fallback requires explicit allow-no-ai-first-publish or force-replace policy")
379
+ elif args.publish_policy == "allow-no-ai-first-publish":
380
+ if published_status.get("good"):
381
+ reasons.append("allow-no-ai-first-publish cannot replace an existing good AI-authored article")
382
+ fallback_errors = fallback_quality_errors(args.summary, validation_passed)
383
+ elif args.publish_policy == "force-replace":
384
+ if not args.force_reason.strip():
385
+ reasons.append("force-replace requires force_reason")
386
+ if not args.actor.strip():
387
+ reasons.append("force-replace requires actor")
388
+ fallback_errors = fallback_quality_errors(args.summary, validation_passed)
389
+ reasons.extend(fallback_errors)
390
+
391
if not candidate_exists:
392
reasons.append(f"candidate summary missing: {args.summary}")
393
if not validation_passed:
394
reasons.append("analysis validation did not pass")
339
- if ai_status != "ai":
395
+ if ai_status not in {"ai", "no-ai"}:
396
reasons.append(f"analysis source is not AI-publishable: {analysis_source or 'unknown'}")
397
reasons.extend(artifact_reasons)
398
reasons.extend(comparison_reasons)
399
400
+ eligible = (
401
+ candidate_exists
402
+ and validation_passed
403
+ and not artifact_reasons
404
+ and not comparison_reasons
405
+ and (
406
+ ai_status == "ai"
407
+ or (ai_status == "no-ai" and args.publish_policy in {"allow-no-ai-first-publish", "force-replace"} and not reasons)
408
+ )
409
+ )
410
preserve_existing = bool(published_status.get("good") and not eligible)
411
decision = "promote" if eligible else "preserve" if preserve_existing else "block"
412
@@ -362,11 +428,30 @@ def create_manifest(args: argparse.Namespace) -> int:
428
"ai_status": ai_status,
429
"source": analysis_source,
430
"model": args.analysis_model,
431
+ "provider": analysis_source,
432
"provenance": {
433
"run_id": args.run_id,
434
"current_datetime": args.current_datetime,
435
+ "authorship": "ai-authored" if ai_status == "ai" else "no-ai-fallback" if ai_status == "no-ai" else "unknown",
436
+ "provider": analysis_source,
437
+ "model": args.analysis_model,
438
+ "fallback_reason": args.fallback_reason.strip() or None,
439
+ "attempted_ai_paths": attempted_ai_paths,
440
},
441
},
442
+ "existing_article": {
443
+ "exists": published_status["exists"],
444
+ "path": published_status["path"],
445
+ "quality_score": published_status.get("quality_score"),
446
+ "provenance": (
447
+ "no-ai-fallback"
448
+ if published_status.get("ai_status") == "no-ai"
449
+ else "ai-authored-assumed"
450
+ if published_status["exists"]
451
+ else "none"
452
+ ),
453
+ "good_ai_authored": bool(published_status.get("good")),
454
+ },
455
"validation": {
456
"status": args.validation_status,
457
"quality_gates": [
@@ -380,8 +465,16 @@ def create_manifest(args: argparse.Namespace) -> int:
465
"promotion": {
466
"eligible": eligible,
467
"decision": decision,
468
+ "policy": args.publish_policy,
469
"reasons": reasons,
470
},
471
+ "audit": {
472
+ "mode": args.publish_policy,
473
+ "actor": args.actor.strip() or None,
474
+ "reason": args.force_reason.strip() or None,
475
+ "source_artifact_count": len(source_artifacts),
476
+ "source_artifacts": source_artifacts,
477
+ },
478
"preservation": {
479
"preserve_existing": preserve_existing,
480
"preserved_summary_path": args.published_summary.as_posix() if preserve_existing else None,
@@ -409,6 +502,21 @@ def assert_eligible(args: argparse.Namespace) -> int:
502
raise SystemExit(f"Publish manifest is missing or malformed: {args.manifest}")
503
if payload.get("schema_version") != SCHEMA_VERSION:
504
raise SystemExit(f"Unsupported publish manifest schema: {payload.get('schema_version')!r}")
505
+ analysis = payload.get("analysis")
506
+ ai_status = analysis.get("ai_status") if isinstance(analysis, dict) else None
507
+ promotion_policy = (payload.get("promotion") or {}).get("policy") if isinstance(payload.get("promotion"), dict) else None
508
+ if ai_status == "no-ai":
509
+ provenance = analysis.get("provenance") if isinstance(analysis, dict) else {}
510
+ if not isinstance(provenance, dict) or provenance.get("authorship") != "no-ai-fallback":
511
+ raise SystemExit("Manifest lacks no-AI fallback provenance.")
512
+ if not provenance.get("fallback_reason"):
513
+ raise SystemExit("Manifest lacks no-AI fallback reason.")
514
+ if not provenance.get("attempted_ai_paths"):
515
+ raise SystemExit("Manifest lacks attempted AI path audit.")
516
+ if promotion_policy == "force-replace":
517
+ audit = payload.get("audit")
518
+ if not isinstance(audit, dict) or not audit.get("actor") or not audit.get("reason"):
519
+ raise SystemExit("Force replacement requires actor and reason in manifest audit.")
520
promotion = payload.get("promotion")
521
if not isinstance(promotion, dict) or promotion.get("eligible") is not True or promotion.get("decision") != "promote":
522
reasons = promotion.get("reasons") if isinstance(promotion, dict) else ["missing promotion block"]
tests/test_analyze_fallback.py
+17
@@ -8,6 +8,7 @@ from unittest import mock
8
from urllib import error
9
10
import scripts.analyze_fallback as analyze_fallback
11
+import scripts.publish_manifest as publish_manifest
12
13
14
class _FakeHTTPResponse(io.BytesIO):
@@ -220,6 +221,22 @@ class AnalyzeFallbackTests(unittest.TestCase):
221
self.assertIn('title: "Ai, Typescript, and This Week\'s Repo Signals"', markdown)
222
self.assertNotIn('title: "Week 23, 2026 Analysis"', markdown)
223
224
+ def test_no_ai_summary_quality_score_stays_below_publication_threshold(self) -> None:
225
+ tests_root = Path(__file__).resolve().parent
226
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
227
+ base = Path(tmpdir)
228
+ raw_path = base / "data" / "raw" / "2026-W23.json"
229
+ raw_path.parent.mkdir(parents=True)
230
+ raw_path.write_text(json.dumps({"week": "2026-W23", "new_repos": [], "trending_repos": []}), encoding="utf-8")
231
+
232
+ markdown = analyze_fallback.generate_no_ai_summary(raw_path, "2026-06-01T09:42:41Z")
233
+
234
+ self.assertLess(
235
+ analyze_fallback.NO_AI_DIAGNOSTIC_QUALITY_SCORE,
236
+ publish_manifest.FALLBACK_MIN_QUALITY_SCORE,
237
+ )
238
+ self.assertIn(f"quality_score: {analyze_fallback.NO_AI_DIAGNOSTIC_QUALITY_SCORE}", markdown)
239
+
240
def test_script_runs_via_python_pathless_invocation(self) -> None:
241
tests_root = Path(__file__).resolve().parent
242
repo_root = tests_root.parent
tests/test_promotion_guard.py
+61
@@ -112,6 +112,26 @@ def manifest_for(root: Path, name: str, **overrides) -> Path:
112
return manifest_path
113
114
115
+def no_ai_manifest_for(root: Path, name: str, *, policy: dict | None = None, quality_score: int = 70) -> Path:
116
+ summary = VALID_REPLACEMENT_SUMMARY.replace("quality_score: 90", f"quality_score: {quality_score}").replace(
117
+ "Better candidate analysis.", "Automated data-only summary generated without AI assistance."
118
+ )
119
+ policy = policy or {"mode": "default"}
120
+ return manifest_for(
121
+ root,
122
+ name,
123
+ summary=summary,
124
+ ai_provenance={
125
+ "source": "no-ai",
126
+ "model": "none",
127
+ "degraded": False,
128
+ "fallback_reason": "copilot quality gate failed",
129
+ "attempted_ai_paths": ["provider=copilot-cli,model=copilot-default,status=failed"],
130
+ },
131
+ promotion_policy=policy,
132
+ )
133
+
134
+
135
class PromotionGuardTests(unittest.TestCase):
136
def test_failed_degraded_and_no_ai_candidates_do_not_replace_existing_good_article(self) -> None:
137
tests_root = Path(__file__).resolve().parent
@@ -191,6 +211,47 @@ class PromotionGuardTests(unittest.TestCase):
211
self.assertEqual(second_summary.read_text(encoding="utf-8").count("Better candidate analysis."), 1)
212
self.assertEqual(second_content.read_text(encoding="utf-8").count("Better candidate rendered content."), 1)
213
214
+ def test_no_ai_first_publish_requires_explicit_policy_and_no_existing_good_article(self) -> None:
215
+ tests_root = Path(__file__).resolve().parent
216
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
217
+ root = Path(tmpdir)
218
+ default_manifest = no_ai_manifest_for(root, "no-ai-default")
219
+
220
+ with self.assertRaises(promotion_guard.PromotionBlocked) as default_block:
221
+ promotion_guard.promote_candidate(default_manifest, root=root)
222
+ self.assertIn("no-AI fallback is ineligible for default promotion.", default_block.exception.reasons)
223
+
224
+ allow_manifest = no_ai_manifest_for(root, "no-ai-first", policy={"mode": "allow-no-ai-first-publish"})
225
+ summary_path, _ = promotion_guard.promote_candidate(allow_manifest, root=root)
226
+
227
+ self.assertIn("Automated data-only summary", summary_path.read_text(encoding="utf-8"))
228
+
229
+ def test_force_replace_no_ai_requires_audit_and_writes_audit_log(self) -> None:
230
+ tests_root = Path(__file__).resolve().parent
231
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
232
+ root = Path(tmpdir)
233
+ canonical_summary, _ = install_existing_good_article(root)
234
+ original_summary = canonical_summary.read_text(encoding="utf-8")
235
+ missing_audit = no_ai_manifest_for(root, "force-missing", policy={"mode": "force-replace"})
236
+
237
+ with self.assertRaises(promotion_guard.PromotionBlocked) as blocked:
238
+ promotion_guard.promote_candidate(missing_audit, root=root)
239
+ self.assertIn("force-replace requires a reason.", blocked.exception.reasons)
240
+ self.assertEqual(canonical_summary.read_text(encoding="utf-8"), original_summary)
241
+
242
+ force_manifest = no_ai_manifest_for(
243
+ root,
244
+ "force-ok",
245
+ policy={"mode": "force-replace", "reason": "operator approved emergency replace", "actor": "jmservera"},
246
+ )
247
+ summary_path, _ = promotion_guard.promote_candidate(force_manifest, root=root)
248
+
249
+ self.assertIn("Automated data-only summary", summary_path.read_text(encoding="utf-8"))
250
+ audit_path = root / "data/diagnostics/promotion/2026-W23-force-replace-audit.json"
251
+ self.assertTrue(audit_path.exists())
252
+ audit = json.loads(audit_path.read_text(encoding="utf-8"))
253
+ self.assertEqual(audit["actor"], "jmservera")
254
+
255
def test_same_day_reused_source_candidate_can_promote_when_manifest_is_fresh(self) -> None:
256
tests_root = Path(__file__).resolve().parent
257
with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
tests/test_publish_manifest.py
+154
-1
@@ -66,6 +66,15 @@ Canonical good analysis.
66
)
67
68
69
+def write_no_ai_summary(path: Path, *, quality_score: int = 70) -> None:
70
+ path.parent.mkdir(parents=True, exist_ok=True)
71
+ path.write_text(
72
+ f"---\nweek: 2026-W21\nquality_score: {quality_score}\nsummary: fallback\n---\n\n"
73
+ "Automated data-only summary generated without AI assistance.\n",
74
+ encoding="utf-8",
75
+ )
76
+
77
+
78
class PublishManifestTests(unittest.TestCase):
79
def test_ai_candidate_with_fresh_sources_is_eligible(self) -> None:
80
tests_root = Path(__file__).resolve().parent
@@ -152,10 +161,154 @@ class PublishManifestTests(unittest.TestCase):
161
payload = json.loads(manifest.read_text(encoding="utf-8"))
162
self.assertFalse(payload["promotion"]["eligible"])
163
self.assertEqual(payload["promotion"]["decision"], "block")
155
- self.assertIn("analysis source is not AI-publishable", payload["promotion"]["reasons"][0])
164
+ self.assertEqual(payload["analysis"]["provenance"]["authorship"], "no-ai-fallback")
165
+ self.assertIn("fallback_reason is required", payload["promotion"]["reasons"][0])
166
with self.assertRaises(SystemExit):
167
publish_manifest.main(["assert-eligible", "--manifest", str(manifest)])
168
169
+ def test_no_ai_default_cannot_replace_existing_good_ai_article(self) -> None:
170
+ tests_root = Path(__file__).resolve().parent
171
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
172
+ base = Path(tmpdir)
173
+ raw = base / "data/raw/2026-W21.json"
174
+ summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
175
+ published = base / "data/analyzed/2026-W21-summary.md"
176
+ manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
177
+ write_raw(raw)
178
+ write_no_ai_summary(summary)
179
+ write_good_summary(published)
180
+
181
+ publish_manifest.main(
182
+ [
183
+ "create",
184
+ "--week", WEEK,
185
+ "--run-id", RUN_ID,
186
+ "--current-datetime", CURRENT_DATETIME,
187
+ "--summary", str(summary),
188
+ "--published-summary", str(published),
189
+ "--raw-json", str(raw),
190
+ "--analysis-source", "no-ai",
191
+ "--analysis-model", "none",
192
+ "--validation-status", "passed",
193
+ "--fallback-reason", "copilot quality gate failed",
194
+ "--attempted-ai-path", "provider=copilot-cli,model=copilot-default,status=failed",
195
+ "--output", str(manifest),
196
+ ]
197
+ )
198
+
199
+ payload = json.loads(manifest.read_text(encoding="utf-8"))
200
+ self.assertFalse(payload["promotion"]["eligible"])
201
+ self.assertEqual(payload["promotion"]["decision"], "preserve")
202
+ self.assertTrue(payload["existing_article"]["good_ai_authored"])
203
+ self.assertIn("no-AI fallback is ineligible to replace", " ".join(payload["promotion"]["reasons"]))
204
+
205
+ def test_no_ai_first_publish_requires_explicit_policy_and_quality_gate(self) -> None:
206
+ tests_root = Path(__file__).resolve().parent
207
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
208
+ base = Path(tmpdir)
209
+ raw = base / "data/raw/2026-W21.json"
210
+ summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
211
+ manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
212
+ write_raw(raw)
213
+ write_no_ai_summary(summary)
214
+
215
+ publish_manifest.main(
216
+ [
217
+ "create",
218
+ "--week", WEEK,
219
+ "--run-id", RUN_ID,
220
+ "--current-datetime", CURRENT_DATETIME,
221
+ "--summary", str(summary),
222
+ "--published-summary", str(base / "data/analyzed/2026-W21-summary.md"),
223
+ "--raw-json", str(raw),
224
+ "--analysis-source", "no-ai",
225
+ "--analysis-model", "none",
226
+ "--validation-status", "passed",
227
+ "--fallback-reason", "copilot unavailable",
228
+ "--attempted-ai-path", "provider=copilot-cli,model=copilot-default,status=failed",
229
+ "--publish-policy", "allow-no-ai-first-publish",
230
+ "--actor", "jmservera",
231
+ "--output", str(manifest),
232
+ ]
233
+ )
234
+
235
+ payload = json.loads(manifest.read_text(encoding="utf-8"))
236
+ self.assertTrue(payload["promotion"]["eligible"])
237
+ self.assertEqual(payload["promotion"]["policy"], "allow-no-ai-first-publish")
238
+ self.assertEqual(publish_manifest.main(["assert-eligible", "--manifest", str(manifest)]), 0)
239
+
240
+ def test_no_ai_explicit_policy_requires_higher_fallback_quality_score(self) -> None:
241
+ tests_root = Path(__file__).resolve().parent
242
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
243
+ base = Path(tmpdir)
244
+ raw = base / "data/raw/2026-W21.json"
245
+ summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
246
+ manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
247
+ write_raw(raw)
248
+ write_no_ai_summary(summary, quality_score=69)
249
+
250
+ publish_manifest.main(
251
+ [
252
+ "create",
253
+ "--week", WEEK,
254
+ "--run-id", RUN_ID,
255
+ "--current-datetime", CURRENT_DATETIME,
256
+ "--summary", str(summary),
257
+ "--published-summary", str(base / "data/analyzed/2026-W21-summary.md"),
258
+ "--raw-json", str(raw),
259
+ "--analysis-source", "no-ai",
260
+ "--analysis-model", "none",
261
+ "--validation-status", "passed",
262
+ "--fallback-reason", "copilot unavailable",
263
+ "--attempted-ai-path", "provider=copilot-cli,model=copilot-default,status=failed",
264
+ "--publish-policy", "allow-no-ai-first-publish",
265
+ "--output", str(manifest),
266
+ ]
267
+ )
268
+
269
+ payload = json.loads(manifest.read_text(encoding="utf-8"))
270
+ self.assertFalse(payload["promotion"]["eligible"])
271
+ self.assertIn("quality_score must be at least 70", " ".join(payload["promotion"]["reasons"]))
272
+
273
+ def test_force_replace_requires_audit_and_allows_no_ai_over_existing_good_article(self) -> None:
274
+ tests_root = Path(__file__).resolve().parent
275
+ with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
276
+ base = Path(tmpdir)
277
+ raw = base / "data/raw/2026-W21.json"
278
+ summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"
279
+ published = base / "data/analyzed/2026-W21-summary.md"
280
+ manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"
281
+ write_raw(raw)
282
+ write_no_ai_summary(summary)
283
+ write_good_summary(published)
284
+
285
+ publish_manifest.main(
286
+ [
287
+ "create",
288
+ "--week", WEEK,
289
+ "--run-id", RUN_ID,
290
+ "--current-datetime", CURRENT_DATETIME,
291
+ "--summary", str(summary),
292
+ "--published-summary", str(published),
293
+ "--raw-json", str(raw),
294
+ "--analysis-source", "no-ai",
295
+ "--analysis-model", "none",
296
+ "--validation-status", "passed",
297
+ "--fallback-reason", "copilot unavailable",
298
+ "--attempted-ai-path", "provider=copilot-cli,model=copilot-default,status=failed",
299
+ "--publish-policy", "force-replace",
300
+ "--force-reason", "operator approved emergency publish",
301
+ "--actor", "jmservera",
302
+ "--output", str(manifest),
303
+ ]
304
+ )
305
+
306
+ payload = json.loads(manifest.read_text(encoding="utf-8"))
307
+ self.assertTrue(payload["promotion"]["eligible"])
308
+ self.assertEqual(payload["audit"]["mode"], "force-replace")
309
+ self.assertEqual(payload["audit"]["actor"], "jmservera")
310
+ self.assertEqual(publish_manifest.main(["assert-eligible", "--manifest", str(manifest)]), 0)
311
+
312
def test_missing_candidate_summary_only_reports_missing_summary(self) -> None:
313
tests_root = Path(__file__).resolve().parent
314
with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir: