@cryptotaxi247 / SquadScope-1 / commits / ba6e192

Add deterministic analysis preflight and fallback policy (#281)

* analysis: add prompt preflight compaction Add deterministic rendered-prompt preflight reporting before Copilot analysis, including component checksums, token bounds, compaction decisions, and Copilot-only fallback classification. Keep no-AI diagnostic-only and remove GitHub Models model wiring from the workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: block degraded preflight promotion Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: enforce copilot-only analysis promotion Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Juan Manuel Servera committed Jun 6, 2026 at 22:10 UTC ba6e192b291ce96bcf61cdcaa1f19cb758f06fb4

10 files changed +873 -120

.github/workflows/crawl-and-publish.yml

+54 -31

@@ -45,7 +45,6 @@ concurrency:

46 env:

47 PYTHONPATH: ${{ github.workspace }}

48 - GITHUB_MODELS_MODEL: ${{ vars.GITHUB_MODELS_MODEL || 'openai/gpt-4o' }}

49 jobs:

50 crawl:

@@ -436,27 +435,65 @@ jobs:

435 PY

436 echo "press_file=$PRESS_FILE" >> "$GITHUB_OUTPUT"

437

439 - - name: Pre-flight cost check

440 - id: preflight-cost

438 + - name: Render and preflight analysis prompt

439 + id: prompt-preflight

440 run: |

441 + set -euo pipefail

442 WEEK_FILE="${{ steps.analysis-context.outputs.week_file }}"

443 - CONTEXT_FILES=("$WEEK_FILE")

444 - if [ -f ".squad/skills/wisdom.md" ]; then

445 - CONTEXT_FILES+=(".squad/skills/wisdom.md")

446 - fi

447 - for f in .squad/skills/*.md; do

448 - [ -f "$f" ] && CONTEXT_FILES+=("$f")

449 - done

450 - # Use the generic Copilot default cost profile so the workflow never pins a stale CLI model.

443 + WEEK="${{ steps.analysis-context.outputs.week }}"

444 + OUTPUT_FILE="${{ steps.analysis-context.outputs.candidate_output_file }}"

445 + CURRENT_DATETIME="${{ steps.analysis-context.outputs.current_datetime }}"

446 + PRESS_FILE="${{ steps.press-context.outputs.press_file }}"

447 + DIAGNOSTICS_DIR="$(dirname "$OUTPUT_FILE")/diagnostics"

448 + PROMPT_FILE="data/metrics/analysis-prompt-${WEEK}.md"

449 + PREFLIGHT_JSON="$DIAGNOSTICS_DIR/analysis-preflight.json"

450 + PREFLIGHT_MD="$DIAGNOSTICS_DIR/analysis-preflight.md"

451 + mkdir -p data/metrics "$(dirname "$OUTPUT_FILE")" "$DIAGNOSTICS_DIR"

452 + # Hydrate metrics ledger from publish before writing this run's prompt/preflight artifacts.

453 + git fetch origin publish 2>/dev/null && \

454 + git checkout origin/publish -- data/metrics/ 2>/dev/null || true

455 + python3 scripts/analyze_fallback.py \

456 + --raw-json "$WEEK_FILE" \

457 + --output "$OUTPUT_FILE" \

458 + --current-datetime "$CURRENT_DATETIME" \

459 + --press-context "$PRESS_FILE" \

460 + --prompt-token-budget "${ANALYSIS_PROMPT_TOKEN_BUDGET:-90000}" \

461 + --preflight-report-json "$PREFLIGHT_JSON" \

462 + --preflight-report-md "$PREFLIGHT_MD" \

463 + --print-prompt > "$PROMPT_FILE"

464 python3 scripts/preflight_cost_check.py \

452 - --context-files "${CONTEXT_FILES[@]}"

465 + --context-files "$PROMPT_FILE"

466 + python3 - <<'PY' "$PREFLIGHT_JSON"

467 + import json

468 + import sys

469 + from pathlib import Path

470 +

471 + report = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))

472 + print(

473 + "::notice::Analysis preflight "

474 + f"tokens={report['prompt_tokens']}/{report['prompt_token_budget']} "

475 + f"bytes={report['prompt_bytes']} "

476 + f"degraded={report['degraded']} "

477 + f"publish_eligible={report['publish_eligible']} "

478 + f"promotion_policy={report.get('promotion_policy', 'unspecified')} "

479 + f"checksum={report['prompt_checksum_sha256'][:12]}"

480 + )

481 + if report.get("degraded") or not report.get("publish_eligible"):

482 + print(

483 + "::warning::Analysis preflight is degraded/compacted or publish-ineligible; "

484 + "output will remain staged/candidate-only unless an explicit promotion policy allows it."

485 + )

486 + PY

487 + echo "prompt_file=$PROMPT_FILE" >> "$GITHUB_OUTPUT"

488 + echo "preflight_report_json=$PREFLIGHT_JSON" >> "$GITHUB_OUTPUT"

489 + echo "preflight_report_md=$PREFLIGHT_MD" >> "$GITHUB_OUTPUT"

490

491 - name: Install Copilot CLI

492 id: install-copilot

493 run: npm install -g @github/copilot

494

495 - name: Run analysis

459 - if: steps.preflight-cost.outcome == 'success'

496 + if: steps.prompt-preflight.outcome == 'success'

497 id: run-analysis

498 env:

499 COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GH_TOKEN }}

@@ -473,18 +510,7 @@ jobs:

510 ANALYSIS_STARTED=$(date +%s)

511 DIAGNOSTICS_DIR="$(dirname "$OUTPUT_FILE")/diagnostics"

512 mkdir -p data/metrics "$(dirname "$OUTPUT_FILE")" "$DIAGNOSTICS_DIR"

476 - # Hydrate metrics ledger from publish so track_token_usage.py appends to

477 - # the canonical token-usage.jsonl rather than starting fresh each run.

478 - git fetch origin publish 2>/dev/null && \

479 - git checkout origin/publish -- data/metrics/ 2>/dev/null || true

480 - PROMPT_FILE="data/metrics/analysis-prompt-${WEEK}.md"

481 - rm -f "$PROMPT_FILE"

482 - python3 scripts/analyze_fallback.py \

483 - --raw-json "$WEEK_FILE" \

484 - --output "$OUTPUT_FILE" \

485 - --current-datetime "$CURRENT_DATETIME" \

486 - --press-context "$PRESS_FILE" \

487 - --print-prompt > "$PROMPT_FILE"

513 + PROMPT_FILE="${{ steps.prompt-preflight.outputs.prompt_file }}"

514

515 sanitize_agent_output() {

516 python3 scripts/sanitize_agent_output.py --path "$1"

@@ -700,6 +726,7 @@ jobs:

726 RUN_MODE: ${{ steps.analysis-context.outputs.run_mode }}

727 SOURCE_REFRESH_POLICY: ${{ steps.analysis-context.outputs.source_refresh_policy }}

728 GATE_REPORT: ${{ steps.analysis-context.outputs.analysis_gate_report_file }}

729 + PREFLIGHT_REPORT: ${{ steps.prompt-preflight.outputs.preflight_report_json }}

730 run: |

731 set -euo pipefail

732 git fetch origin publish 2>/dev/null && git checkout origin/publish -- "$PUBLISHED_SUMMARY" 2>/dev/null || true

@@ -722,6 +749,7 @@ jobs:

749 --raw-json "$RAW_JSON_FILE" \

750 --analysis-source "$ANALYSIS_SOURCE" \

751 --analysis-model "$ANALYSIS_MODEL" \

752 + --preflight-report "$PREFLIGHT_REPORT" \

753 --validation-status "$VALIDATION_STATUS" \

754 --run-mode "$RUN_MODE" \

755 --source-refresh-policy "$SOURCE_REFRESH_POLICY" \

@@ -1320,16 +1348,11 @@ jobs:

1348 --allow-tool=grep \

1349 > /dev/null; then

1350 echo "✅ Reskill via Copilot CLI with agent identity"

1323 - # Fallback: GitHub Models API via reskill.py

1324 - elif python3 scripts/reskill.py --current-datetime "$CURRENT_DATETIME" --output "$RESKILL_OUTPUT" --prompt-output "$RESKILL_PROMPT"; then

1325 - RESKILL_SOURCE="github-models"

1326 - RESKILL_MODEL="${GITHUB_MODELS_MODEL}"

1327 - echo "⚠️ Copilot CLI unavailable; used GitHub Models API fallback."

1351 else

1352 RESKILL_SOURCE="none"

1353 RESKILL_MODEL="none"

1354 rm -f "$RESKILL_PROMPT"

1332 - echo "Reskill failed on all paths; writing placeholder trigger log."

1355 + echo "Copilot CLI reskill failed; no GitHub Models/OpenAI reskill fallback is configured. Writing placeholder trigger log."

1356 echo "Reskill triggered at run #$COUNTER ($CURRENT_DATETIME)" >> .squad/reskill/trigger-log.txt

1357 fi

1358

prompts/analyze-weekly.md

+2 -2

@@ -35,8 +35,8 @@ Use this only if it is provided. If it is missing, unavailable, or empty, say so

36 The analyze job must resolve both learned-state placeholders before invoking Copilot CLI. Weekly AI analysis is Copilot-only; there is no GitHub Models/OpenAI fallback configured for this repository.

38 -1. Read `.squad/identity/wisdom.md` and inject its current contents into `{{WISDOM}}`.

39 -2. Read markdown files under `.squad/skills/` (for example `SKILL.md` files in nested skill folders), concatenate them in a stable sorted order, and inject that bundle into `{{SKILLS}}`.

38 +1. Inject only the analysis/topic-specific wisdom capsule into `{{WISDOM}}` (for this topic, the `.squad/topics/<topic>/wisdom.md` learning state or configured equivalent).

39 +2. Inject only analysis/topic-specific skill markdown into `{{SKILLS}}`, in stable sorted order. Do not include unrelated squad workflow, UI, PR-review, or release-process skills.

40 3. If either source is missing or empty, inject a short explicit note rather than leaving the placeholder unresolved.

41 4. Treat learned context as guidance that sharpens judgment, not as permission to ignore the current week's evidence.

scripts/analyze_fallback.py

+426 -33

@@ -2,11 +2,13 @@

2 from __future__ import annotations

4 import argparse

5 +import hashlib

6 import json

7 import os

8 import random

9 import sys

10 import time

11 +from dataclasses import asdict, dataclass

12 from pathlib import Path

13 from typing import Any

14 from urllib import error, request

@@ -26,10 +28,45 @@ DEFAULT_MODELS_ENDPOINT = "https://models.github.ai/inference/chat/completions"

28 DEFAULT_MODELS_MODEL = "openai/gpt-4o"

29 DEFAULT_MODELS_TIMEOUT = 30

30 NO_AI_DIAGNOSTIC_QUALITY_SCORE = 40

31 +DEFAULT_PROMPT_TOKEN_BUDGET = 90_000

32 +COMPACTED_NEW_REPOS_LIMIT = 25

33 +COMPACTED_TRENDING_REPOS_LIMIT = 25

34 +COMPACTED_PREVIOUS_SUMMARY_CHARS = 8_000

35 +COMPACTED_WISDOM_CHARS = 8_000

36 +COMPACTED_SKILLS_CHARS = 10_000

37 +COMPACTED_PRESS_CONTEXT_CHARS = 14_000

38 +

39 +

40 +@dataclass

41 +class PromptComponent:

42 + name: str

43 + path: str | None

44 + included: bool

45 + inclusion_reason: str

46 + compaction_decision: str

47 + bytes: int

48 + token_estimate: int

49 + checksum_sha256: str

50 +

51 +

52 +@dataclass

53 +class PromptPreflight:

54 + prompt_token_budget: int

55 + prompt_tokens: int

56 + prompt_bytes: int

57 + prompt_checksum_sha256: str

58 + prompt_within_budget: bool

59 + degraded: bool

60 + publish_eligible: bool

61 + promotion_policy: str

62 + degradation_reason: str | None

63 + fallback_policy: str

64 + components: list[PromptComponent]

65 + deterministic_slices: list[str]

68 def parse_args(argv: list[str] | None = None) -> argparse.Namespace:

32 - parser = argparse.ArgumentParser(description="Fallback weekly analysis via GitHub Models API.")

69 + parser = argparse.ArgumentParser(description="Render/preflight weekly analysis prompts or generate diagnostic no-AI output.")

70 parser.add_argument("--raw-json", required=True, type=Path, help="Path to the weekly raw JSON payload.")

71 parser.add_argument("--output", required=True, type=Path, help="Path to write the analyzed markdown output.")

72 parser.add_argument("--current-datetime", required=True, help="ISO-8601 timestamp for the analysis run.")

@@ -73,6 +110,22 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:

110 action="store_true",

111 help="Generate a data-only summary without calling any AI API.",

112 )

113 + parser.add_argument(

114 + "--prompt-token-budget",

115 + type=int,

116 + default=DEFAULT_PROMPT_TOKEN_BUDGET,

117 + help=f"Maximum rendered prompt tokens before model invocation (default: {DEFAULT_PROMPT_TOKEN_BUDGET}).",

118 + )

119 + parser.add_argument(

120 + "--preflight-report-json",

121 + type=Path,

122 + help="Write deterministic rendered-prompt preflight details as JSON.",

123 + )

124 + parser.add_argument(

125 + "--preflight-report-md",

126 + type=Path,

127 + help="Write deterministic rendered-prompt preflight details as Markdown.",

128 + )

129 return parser.parse_args(argv)

130

131

@@ -80,6 +133,90 @@ def load_json(path: Path) -> dict[str, Any]:

133 return json.loads(path.read_text(encoding="utf-8"))

134

135

136 +def estimate_tokens(text: str) -> int:

137 + """Deterministic local estimate used for preflight bounds."""

138 + return (len(text.encode("utf-8")) + 3) // 4

139 +

140 +

141 +def checksum_text(text: str) -> str:

142 + return hashlib.sha256(text.encode("utf-8")).hexdigest()

143 +

144 +

145 +def _component(

146 + *,

147 + name: str,

148 + content: str,

149 + path: Path | None,

150 + included: bool,

151 + inclusion_reason: str,

152 + compaction_decision: str,

153 +) -> PromptComponent:

154 + return PromptComponent(

155 + name=name,

156 + path=path.as_posix() if path else None,

157 + included=included,

158 + inclusion_reason=inclusion_reason,

159 + compaction_decision=compaction_decision,

160 + bytes=len(content.encode("utf-8")),

161 + token_estimate=estimate_tokens(content),

162 + checksum_sha256=checksum_text(content),

163 + )

164 +

165 +

166 +def truncate_with_notice(content: str, limit: int, label: str) -> tuple[str, str]:

167 + if len(content) <= limit:

168 + return content, "included"

169 + omitted = len(content) - limit

170 + return (

171 + content[:limit].rstrip()

172 + + f"\n\n[Preflight compaction: truncated {label}; omitted {omitted} characters to stay within prompt budget.]",

173 + "compacted",

174 + )

175 +

176 +

177 +def _load_yaml(path: Path) -> dict[str, Any]:

178 + try:

179 + import yaml # type: ignore[import-untyped]

180 + except ImportError:

181 + return {}

182 + if not path.exists():

183 + return {}

184 + with open(path, encoding="utf-8") as f:

185 + payload = yaml.safe_load(f) or {}

186 + return payload if isinstance(payload, dict) else {}

187 +

188 +

189 +def _resolve_existing_path(configured: str | None, fallback: Path) -> Path:

190 + candidates: list[Path] = []

191 + if configured:

192 + configured_path = Path(configured)

193 + candidates.append(configured_path if configured_path.is_absolute() else ROOT / configured_path)

194 + if not configured_path.is_absolute():

195 + candidates.append(ROOT / ".squad" / configured_path)

196 + candidates.append(fallback)

197 + for candidate in candidates:

198 + if candidate.exists():

199 + return candidate

200 + return candidates[0] if candidates else fallback

201 +

202 +

203 +def resolve_analysis_context_paths() -> tuple[Path, Path]:

204 + """Resolve analysis-specific learned context, avoiding unrelated squad workflow context."""

205 + config = _load_yaml(ROOT / "squadscope.topic.yml")

206 + topic = config.get("topic") if isinstance(config.get("topic"), dict) else {}

207 + learning = config.get("learning") if isinstance(config.get("learning"), dict) else {}

208 + topic_id = str(topic.get("id") or "general")

209 + wisdom_path = _resolve_existing_path(

210 + learning.get("wisdom_file"),

211 + ROOT / ".squad" / "topics" / topic_id / "wisdom.md",

212 + )

213 + skills_path = _resolve_existing_path(

214 + learning.get("skills_dir"),

215 + ROOT / ".squad" / "topics" / topic_id / "skills",

216 + )

217 + return wisdom_path, skills_path

218 +

219 +

220 def find_previous_summary(current_week: str, analyzed_dir: Path) -> Path | None:

221 if not analyzed_dir.exists():

222 return None

@@ -121,7 +258,43 @@ def render_skills(skills_dir: Path) -> str:

258 return "\n\n".join(blocks) if blocks else "_No learned skills have been extracted yet._"

259

260

124 -def render_prompt(

261 +def _sort_repos_for_compaction(repos: list[dict[str, Any]], score_key: str) -> list[dict[str, Any]]:

262 + return sorted(

263 + repos,

264 + key=lambda repo: (

265 + int(repo.get(score_key) or 0),

266 + int(repo.get("stars") or 0),

267 + str(repo.get("full_name") or ""),

268 + ),

269 + reverse=True,

270 + )

271 +

272 +

273 +def compact_payload(payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, str]]:

274 + compacted = dict(payload)

275 + decisions = {"new_repos": "included", "trending_repos": "included"}

276 + new_repos = payload.get("new_repos")

277 + if isinstance(new_repos, list) and len(new_repos) > COMPACTED_NEW_REPOS_LIMIT:

278 + compacted["new_repos"] = _sort_repos_for_compaction(new_repos, "stars")[:COMPACTED_NEW_REPOS_LIMIT]

279 + decisions["new_repos"] = f"compacted to top {COMPACTED_NEW_REPOS_LIMIT} repos by stars"

280 + trending_repos = payload.get("trending_repos")

281 + if isinstance(trending_repos, list) and len(trending_repos) > COMPACTED_TRENDING_REPOS_LIMIT:

282 + compacted["trending_repos"] = _sort_repos_for_compaction(trending_repos, "stars_gained")[

283 + :COMPACTED_TRENDING_REPOS_LIMIT

284 + ]

285 + decisions["trending_repos"] = f"compacted to top {COMPACTED_TRENDING_REPOS_LIMIT} repos by stars_gained/stars"

286 + if decisions["new_repos"] != "included" or decisions["trending_repos"] != "included":

287 + compacted["_preflight_compaction"] = {

288 + "reason": "Rendered prompt exceeded explicit token budget before model invocation.",

289 + "new_repos_original_count": len(new_repos) if isinstance(new_repos, list) else 0,

290 + "trending_repos_original_count": len(trending_repos) if isinstance(trending_repos, list) else 0,

291 + "new_repos_decision": decisions["new_repos"],

292 + "trending_repos_decision": decisions["trending_repos"],

293 + }

294 + return compacted, decisions

295 +

296 +

297 +def _build_prompt(

298 *,

299 prompt_template_path: Path,

300 raw_json_path: Path,

@@ -131,44 +304,244 @@ def render_prompt(

304 wisdom_file: Path = DEFAULT_WISDOM_FILE,

305 skills_dir: Path = DEFAULT_SKILLS_DIR,

306 press_context_path: Path | None = None,

134 -) -> str:

307 + prompt_token_budget: int = DEFAULT_PROMPT_TOKEN_BUDGET,

308 + allow_compaction: bool = True,

309 +) -> tuple[str, PromptPreflight]:

310 payload = load_json(raw_json_path)

311 sanitized_payload = sanitize_repo_payload(payload)

312 current_week = sanitized_payload["week"]

313 previous_summary_path = find_previous_summary(current_week, analyzed_dir)

314 previous_summary_content = previous_summary_path.read_text(encoding="utf-8") if previous_summary_path else ""

140 - raw_json_content = json.dumps(sanitized_payload, indent=2, ensure_ascii=False)

315 + wisdom_content = render_wisdom(wisdom_file)

316 + skills_content = render_skills(skills_dir)

317 + press_content = (

318 + press_context_path.read_text(encoding="utf-8").strip()

319 + if press_context_path and press_context_path.exists() and press_context_path.stat().st_size > 0

320 + else ""

321 + )

322 + payload_for_prompt = sanitized_payload

323 + raw_decisions = {"new_repos": "included", "trending_repos": "included"}

324 + previous_decision = "included" if previous_summary_path else "not included: no previous summary"

325 + wisdom_decision = "included" if wisdom_file.exists() else "not included: no analysis-specific wisdom file"

326 + skills_decision = "included" if skills_dir.exists() and iter_skill_files(skills_dir) else "not included: no analysis-specific skills"

327 + press_decision = "included" if press_content else "not included: no press context"

328 + degraded = False

329 +

330 + def assemble() -> str:

331 + raw_json_content = json.dumps(payload_for_prompt, indent=2, ensure_ascii=False)

332 + current_year, _, week_number = current_week.partition("-W")

333 + generic_title_example = (

334 + f"Week {int(week_number)}, {current_year} Analysis" if week_number.isdigit() else "Week NN, YYYY Analysis"

335 + )

336 + prompt = prompt_template_path.read_text(encoding="utf-8")

337 + replacements = {

338 + "{{CURRENT_DATETIME}}": current_datetime,

339 + "{{CURRENT_WEEK}}": current_week,

340 + "{{CURRENT_YEAR}}": current_year,

341 + "{{TITLE_TEMPLATE_HINT}}": (

342 + f"Specific editorial headline about {current_week}'s dominant themes "

343 + f"(not \"{generic_title_example}\")"

344 + ),

345 + "{{RAW_JSON_PATH}}": str(raw_json_path),

346 + "{{OUTPUT_PATH}}": str(output_path),

347 + "{{PREVIOUS_SUMMARY_PATH_OR_NONE}}": str(previous_summary_path) if previous_summary_path else "None",

348 + "{{RAW_JSON_CONTENT}}": raw_json_content,

349 + "{{PREVIOUS_SUMMARY_CONTENT_OR_EMPTY}}": previous_summary_content.strip(),

350 + "{{WISDOM}}": wisdom_content,

351 + "{{SKILLS}}": skills_content,

352 + }

353 + for needle, value in replacements.items():

354 + prompt = prompt.replace(needle, value)

355 + if press_content:

356 + prompt += f"\n\n---\n## Press Context\n\n{press_content}\n"

357 + return prompt

358 +

359 + prompt = assemble()

360 + if allow_compaction and estimate_tokens(prompt) > prompt_token_budget:

361 + degraded = True

362 + payload_for_prompt, raw_decisions = compact_payload(sanitized_payload)

363 + previous_summary_content, previous_decision = truncate_with_notice(

364 + previous_summary_content, COMPACTED_PREVIOUS_SUMMARY_CHARS, "prior continuity"

365 + )

366 + wisdom_content, wisdom_decision = truncate_with_notice(wisdom_content, COMPACTED_WISDOM_CHARS, "analysis wisdom")

367 + skills_content, skills_decision = truncate_with_notice(skills_content, COMPACTED_SKILLS_CHARS, "analysis skills")

368 + press_content, press_decision = truncate_with_notice(

369 + press_content, COMPACTED_PRESS_CONTEXT_CHARS, "press correlations"

370 + )

371 + prompt = assemble()

372 +

373 + raw_json_content = json.dumps(payload_for_prompt, indent=2, ensure_ascii=False)

374 current_year, _, week_number = current_week.partition("-W")

142 - generic_title_example = f"Week {int(week_number)}, {current_year} Analysis" if week_number.isdigit() else "Week NN, YYYY Analysis"

143 -

144 - prompt = prompt_template_path.read_text(encoding="utf-8")

145 - replacements = {

146 - "{{CURRENT_DATETIME}}": current_datetime,

147 - "{{CURRENT_WEEK}}": current_week,

148 - "{{CURRENT_YEAR}}": current_year,

149 - "{{TITLE_TEMPLATE_HINT}}": (

150 - f"Specific editorial headline about {current_week}'s dominant themes "

151 - f"(not \"{generic_title_example}\")"

375 + components = [

376 + _component(

377 + name="prompt_template",

378 + content=prompt_template_path.read_text(encoding="utf-8"),

379 + path=prompt_template_path,

380 + included=True,

381 + inclusion_reason="Base weekly analysis instructions.",

382 + compaction_decision="included",

383 ),

153 - "{{RAW_JSON_PATH}}": str(raw_json_path),

154 - "{{OUTPUT_PATH}}": str(output_path),

155 - "{{PREVIOUS_SUMMARY_PATH_OR_NONE}}": str(previous_summary_path) if previous_summary_path else "None",

156 - "{{RAW_JSON_CONTENT}}": raw_json_content,

157 - "{{PREVIOUS_SUMMARY_CONTENT_OR_EMPTY}}": previous_summary_content.strip(),

158 - "{{WISDOM}}": render_wisdom(wisdom_file),

159 - "{{SKILLS}}": render_skills(skills_dir),

160 - }

161 - for needle, value in replacements.items():

162 - prompt = prompt.replace(needle, value)

384 + _component(

385 + name="new_repos",

386 + content=json.dumps(payload_for_prompt.get("new_repos", []), indent=2, ensure_ascii=False),

387 + path=raw_json_path,

388 + included=True,

389 + inclusion_reason="Deterministic mapper slice: newly discovered repositories.",

390 + compaction_decision=raw_decisions["new_repos"],

391 + ),

392 + _component(

393 + name="trending_repos",

394 + content=json.dumps(payload_for_prompt.get("trending_repos", []), indent=2, ensure_ascii=False),

395 + path=raw_json_path,

396 + included=True,

397 + inclusion_reason="Deterministic mapper slice: continuing/trending repositories.",

398 + compaction_decision=raw_decisions["trending_repos"],

399 + ),

400 + _component(

401 + name="raw_metadata",

402 + content=raw_json_content,

403 + path=raw_json_path,

404 + included=True,

405 + inclusion_reason=f"Sanitized current weekly payload for {current_year}-W{week_number}.",

406 + compaction_decision="included" if not degraded else "included with compacted repo slices",

407 + ),

408 + _component(

409 + name="prior_continuity",

410 + content=previous_summary_content,

411 + path=previous_summary_path,

412 + included=bool(previous_summary_path),

413 + inclusion_reason="Deterministic mapper slice: prior weekly continuity.",

414 + compaction_decision=previous_decision,

415 + ),

416 + _component(

417 + name="analysis_wisdom",

418 + content=wisdom_content,

419 + path=wisdom_file,

420 + included=wisdom_file.exists(),

421 + inclusion_reason="Analysis-specific wisdom capsule from topic learning state.",

422 + compaction_decision=wisdom_decision,

423 + ),

424 + _component(

425 + name="analysis_skills",

426 + content=skills_content,

427 + path=skills_dir,

428 + included=skills_dir.exists() and bool(iter_skill_files(skills_dir)),

429 + inclusion_reason="Analysis-specific learned skill capsule from topic learning state.",

430 + compaction_decision=skills_decision,

431 + ),

432 + _component(

433 + name="press_correlations",

434 + content=press_content,

435 + path=press_context_path,

436 + included=bool(press_content),

437 + inclusion_reason="Deterministic mapper slice: press/developer correlation context.",

438 + compaction_decision=press_decision,

439 + ),

440 + _component(

441 + name="rendered_prompt",

442 + content=prompt,

443 + path=None,

444 + included=True,

445 + inclusion_reason="Exact prompt that will be passed to Copilot CLI.",

446 + compaction_decision="included" if not degraded else "included after deterministic compaction",

447 + ),

448 + ]

449 + prompt_tokens = estimate_tokens(prompt)

450 + prompt_within_budget = prompt_tokens <= prompt_token_budget

451 + degradation_reason = (

452 + "Prompt was deterministically compacted to fit the configured token budget." if degraded else None

453 + )

454 + preflight = PromptPreflight(

455 + prompt_token_budget=prompt_token_budget,

456 + prompt_tokens=prompt_tokens,

457 + prompt_bytes=len(prompt.encode("utf-8")),

458 + prompt_checksum_sha256=checksum_text(prompt),

459 + prompt_within_budget=prompt_within_budget,

460 + degraded=degraded,

461 + publish_eligible=prompt_within_budget and not degraded,

462 + promotion_policy=(

463 + "normal-promotion"

464 + if not degraded

465 + else "staged/candidate-only by default; degraded compacted output requires an explicit future promotion policy."

466 + ),

467 + degradation_reason=degradation_reason,

468 + fallback_policy=(

469 + "copilot-only; no GitHub Models/OpenAI fallback. no-ai is diagnostic/staged-only and publish-ineligible. "

470 + "degraded/compacted prompts are staged/candidate-only by default."

471 + ),

472 + components=components,

473 + deterministic_slices=["new_repos", "trending_repos", "press_correlations", "prior_continuity"],

474 + )

475 + return prompt, preflight

476

164 - # Append press context if available

165 - if press_context_path and press_context_path.exists() and press_context_path.stat().st_size > 0:

166 - press_content = press_context_path.read_text(encoding="utf-8").strip()

167 - prompt += f"\n\n---\n## Press Context\n\n{press_content}\n"

477

478 +def render_prompt(

479 + *,

480 + prompt_template_path: Path,

481 + raw_json_path: Path,

482 + output_path: Path,

483 + current_datetime: str,

484 + analyzed_dir: Path,

485 + wisdom_file: Path = DEFAULT_WISDOM_FILE,

486 + skills_dir: Path = DEFAULT_SKILLS_DIR,

487 + press_context_path: Path | None = None,

488 +) -> str:

489 + prompt, _ = _build_prompt(

490 + prompt_template_path=prompt_template_path,

491 + raw_json_path=raw_json_path,

492 + output_path=output_path,

493 + current_datetime=current_datetime,

494 + analyzed_dir=analyzed_dir,

495 + wisdom_file=wisdom_file,

496 + skills_dir=skills_dir,

497 + press_context_path=press_context_path,

498 + allow_compaction=False,

499 + )

500 return prompt

501

502

503 +def write_preflight_reports(preflight: PromptPreflight, json_path: Path | None, md_path: Path | None) -> None:

504 + if json_path:

505 + json_path.parent.mkdir(parents=True, exist_ok=True)

506 + json_path.write_text(json.dumps(asdict(preflight), indent=2, sort_keys=True) + "\n", encoding="utf-8")

507 + if md_path:

508 + md_path.parent.mkdir(parents=True, exist_ok=True)

509 + rows = [

510 + "# Analysis Prompt Preflight",

511 + "",

512 + f"- Prompt budget: `{preflight.prompt_token_budget}` tokens",

513 + f"- Rendered prompt: `{preflight.prompt_tokens}` tokens / `{preflight.prompt_bytes}` bytes",

514 + f"- Prompt checksum: `{preflight.prompt_checksum_sha256}`",

515 + f"- Degraded/compacted: `{str(preflight.degraded).lower()}`",

516 + f"- Degradation reason: {preflight.degradation_reason or 'none'}",

517 + f"- Publish eligible: `{str(preflight.publish_eligible).lower()}`",

518 + f"- Promotion policy: {preflight.promotion_policy}",

519 + f"- Fallback policy: {preflight.fallback_policy}",

520 + f"- Deterministic slices: {', '.join(preflight.deterministic_slices)}",

521 + "",

523 + "| --- | --- | ---: | ---: | --- | --- | --- | --- |",

524 + ]

525 + for component in preflight.components:

526 + rows.append(

527 + "| "

528 + + " | ".join(

529 + [

530 + component.name,

531 + str(component.included).lower(),

532 + str(component.bytes),

533 + str(component.token_estimate),

534 + component.checksum_sha256,

535 + component.path or "",

536 + component.inclusion_reason.replace("|", "\\|"),

537 + component.compaction_decision.replace("|", "\\|"),

538 + ]

539 + )

540 + + " |"

541 + )

542 + md_path.write_text("\n".join(rows) + "\n", encoding="utf-8")

543 +

544 +

545 def extract_markdown(response_payload: dict[str, Any]) -> str:

546 choices = response_payload.get("choices") or []

547 if not choices:

@@ -514,25 +887,45 @@ Week {week_num} of {year_str} captured {repos_featured} repositories with {total

887

888 def main(argv: list[str] | None = None) -> int:

889 args = parse_args(argv)

517 - prompt = render_prompt(

890 + wisdom_file = args.wisdom_file

891 + skills_dir = args.skills_dir

892 + if wisdom_file == DEFAULT_WISDOM_FILE and skills_dir == DEFAULT_SKILLS_DIR:

893 + wisdom_file, skills_dir = resolve_analysis_context_paths()

894 +

895 + prompt, preflight = _build_prompt(

896 prompt_template_path=args.prompt_template,

897 raw_json_path=args.raw_json,

898 output_path=args.output,

899 current_datetime=args.current_datetime,

900 analyzed_dir=args.analyzed_dir,

523 - wisdom_file=args.wisdom_file,

524 - skills_dir=args.skills_dir,

901 + wisdom_file=wisdom_file,

902 + skills_dir=skills_dir,

903 press_context_path=args.press_context,

904 + prompt_token_budget=args.prompt_token_budget,

905 + allow_compaction=True,

906 )

907 + write_preflight_reports(preflight, args.preflight_report_json, args.preflight_report_md)

908 +

909 + if not preflight.prompt_within_budget:

910 + print(

911 + "::error::Rendered analysis prompt exceeds explicit budget after deterministic compaction: "

912 + f"{preflight.prompt_tokens}/{preflight.prompt_token_budget} tokens.",

913 + file=sys.stderr,

914 + )

915 + return 1

916

917 if args.print_prompt:

529 - print(prompt)

918 + sys.stdout.write(prompt)

919 return 0

920

921 if args.no_ai:

922 markdown = generate_no_ai_summary(args.raw_json, args.current_datetime, args.press_context)

923 else:

535 - markdown = call_github_models(prompt)

924 + print(

925 + "::error::GitHub Models/OpenAI analysis fallback is disabled; use Copilot CLI or --no-ai for staged diagnostics.",

926 + file=sys.stderr,

927 + )

928 + return 1

929

930 args.output.parent.mkdir(parents=True, exist_ok=True)

931 args.output.write_text(markdown, encoding="utf-8")

scripts/copilot_failure.py

+10 -2

@@ -62,6 +62,14 @@ TRANSIENT_PATTERNS = (

63 def classify_log(log_text: str, exit_code: int | None = None) -> CopilotFailure:

64 normalized = log_text.lower()

65 + if "copilot is not available" in normalized or "command not found" in normalized:

66 + return CopilotFailure(

67 + "copilot_inaccessible",

68 + retryable=False,

69 + actionable=True,

70 + diagnostic="Copilot CLI is unavailable in the runner; install Copilot CLI or verify runner access.",

71 + exit_code=exit_code,

72 + )

73 if any(pattern in normalized for pattern in TOKEN_PATTERNS):

74 return CopilotFailure(

75 "copilot_token_failure",

@@ -96,10 +104,10 @@ def classify_log(log_text: str, exit_code: int | None = None) -> CopilotFailure:

104 )

105 if any(pattern in normalized for pattern in INACCESSIBLE_PATTERNS):

106 return CopilotFailure(

99 - "copilot_inaccessible",

107 + "copilot_token_failure",

108 retryable=False,

109 actionable=True,

102 - diagnostic="Copilot is inaccessible for this workflow; verify Copilot availability and permissions.",

110 + diagnostic="Copilot authentication/access failure; renew COPILOT_GH_TOKEN or verify Copilot permissions.",

111 exit_code=exit_code,

112 )

113 return CopilotFailure(

scripts/publish_manifest.py

+44 -1

@@ -11,7 +11,7 @@ from typing import Any

13 SCHEMA_VERSION = "publish_eligibility_v1"

14 -AI_SOURCES = {"copilot-cli", "github-models"}

14 +AI_SOURCES = {"copilot-cli"}

15 RUN_MODES = {"normal", "dry-run", "restore", "force-replace", "candidate-only"}

16 SOURCE_REFRESH_POLICIES = {"reuse-same-day", "refresh-missing-stale", "force-refresh"}

17 ALLOWED_PROMOTION_MANIFEST_ROOTS = {("data", "staging"), ("data", "candidates")}

@@ -42,6 +42,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:

42 create.add_argument("--raw-json", required=True, type=Path)

43 create.add_argument("--analysis-source", required=True)

44 create.add_argument("--analysis-model", default="copilot-default")

45 + create.add_argument(

46 + "--preflight-report",

47 + type=Path,

48 + help="Analysis preflight report JSON used to decide whether Copilot output is normally promotable.",

49 + )

50 create.add_argument("--validation-status", choices=["passed", "failed"], required=True)

51 create.add_argument("--run-mode", choices=sorted(RUN_MODES), default="normal")

52 create.add_argument("--source-refresh-policy", choices=sorted(SOURCE_REFRESH_POLICIES), default="reuse-same-day")

@@ -111,6 +116,24 @@ def load_json(path: Path) -> dict[str, Any] | None:

116 return payload if isinstance(payload, dict) else None

117

118

119 +def load_preflight(path: Path | None, *, required: bool = False) -> tuple[dict[str, Any] | None, list[str]]:

120 + if path is None:

121 + if required:

122 + return None, ["preflight report is required for Copilot CLI promotion"]

123 + return None, []

124 + payload = load_json(path)

125 + if payload is None:

126 + return None, [f"preflight report missing or malformed: {path}"]

127 + reasons: list[str] = []

128 + if payload.get("degraded") is True:

129 + reasons.append(

130 + "preflight degraded/compacted; candidate is staged-only unless an explicit promotion policy allows it"

131 + )

132 + if payload.get("publish_eligible") is not True:

133 + reasons.append("preflight report marks candidate as publish-ineligible")

134 + return payload, reasons

135 +

136 +

137 def manifest_lives_under_allowed_promotion_root(manifest_path: Path, root: Path | None = None) -> bool:

138 workspace = (root or Path.cwd()).resolve()

139 resolved_manifest = manifest_path if manifest_path.is_absolute() else workspace / manifest_path

@@ -463,6 +486,7 @@ def create_manifest(args: argparse.Namespace) -> int:

486 analysis_source = args.analysis_source.strip()

487 ai_status = "ai" if analysis_source in AI_SOURCES else "no-ai" if analysis_source == NO_AI_SOURCE else "unknown"

488 model_status = publishable_model_status(args.analysis_model)

489 + preflight, preflight_reasons = load_preflight(args.preflight_report, required=ai_status == "ai")

490 gate_report = load_gate_report(args.gate_report)

491 candidate_metadata = markdown_metadata(args.summary)

492 published_status = published_summary_status(args.published_summary, args.week)

@@ -524,6 +548,7 @@ def create_manifest(args: argparse.Namespace) -> int:

548 reasons.append("analysis validation did not pass")

549 if ai_status not in {"ai", "no-ai"}:

550 reasons.append(f"analysis source is not AI-publishable: {analysis_source or 'unknown'}")

551 + reasons.extend(preflight_reasons)

552 if not mode_allows_promotion:

553 reasons.append(f"run mode {args.run_mode} is non-publishing")

554 if ai_status == "ai" and model_status != "available":

@@ -540,6 +565,7 @@ def create_manifest(args: argparse.Namespace) -> int:

565 and gates_passed

566 and not artifact_reasons

567 and not comparison_reasons

568 + and not preflight_reasons

569 and mode_allows_promotion

570 and not reasons

571 and (

@@ -577,12 +603,23 @@ def create_manifest(args: argparse.Namespace) -> int:

603 "model": args.analysis_model,

604 "model_status": model_status,

605 "provider": analysis_source,

606 + "preflight": {

607 + "path": args.preflight_report.as_posix() if args.preflight_report else None,

608 + "degraded": preflight.get("degraded") if preflight else None,

609 + "publish_eligible": preflight.get("publish_eligible") if preflight else None,

610 + "prompt_tokens": preflight.get("prompt_tokens") if preflight else None,

611 + "prompt_token_budget": preflight.get("prompt_token_budget") if preflight else None,

612 + "prompt_checksum_sha256": preflight.get("prompt_checksum_sha256") if preflight else None,

613 + "promotion_policy": preflight.get("promotion_policy") if preflight else None,

614 + "degradation_reason": preflight.get("degradation_reason") if preflight else None,

615 + },

616 "provenance": {

617 "run_id": args.run_id,

618 "current_datetime": args.current_datetime,

619 "authorship": "ai-authored" if ai_status == "ai" else "no-ai-fallback" if ai_status == "no-ai" else "unknown",

620 "provider": analysis_source,

621 "model": args.analysis_model,

622 + "degraded": preflight.get("degraded") if preflight else None,

623 "fallback_reason": args.fallback_reason.strip() or None,

624 "attempted_ai_paths": attempted_ai_paths,

625 },

@@ -674,6 +711,12 @@ def assert_eligible(args: argparse.Namespace) -> int:

711 ai_status = analysis.get("ai_status")

712 if ai_status == "ai" and analysis.get("model_status") != "available":

713 raise SystemExit("Manifest lacks an available AI model.")

714 + if ai_status == "ai":

715 + preflight = analysis.get("preflight")

716 + if not isinstance(preflight, dict) or preflight.get("publish_eligible") is not True:

717 + raise SystemExit("Manifest lacks a publish-eligible Copilot preflight report.")

718 + if preflight.get("degraded") is True:

719 + raise SystemExit("Manifest preflight is degraded/compacted and staged-only by default.")

720 validation = payload.get("validation")

721 gate_report = validation.get("gate_report") if isinstance(validation, dict) else None

722 if not isinstance(gate_report, dict) or gate_report.get("present") is not True or gate_report.get("passed") is not True:

tests/test_analyze_fallback.py

+127 -11

@@ -180,6 +180,125 @@ class AnalyzeFallbackTests(unittest.TestCase):

180 self.assertNotIn("{{WISDOM}}", prompt)

181 self.assertNotIn("{{SKILLS}}", prompt)

182

183 + def test_main_writes_prompt_preflight_report_for_exact_rendered_prompt(self) -> None:

184 + tests_root = Path(__file__).resolve().parent

185 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

186 + base = Path(tmpdir)

187 + raw_path = base / "data" / "raw" / "2026-W21.json"

188 + prompt_template = base / "prompt.md"

189 + output_path = base / "data" / "analyzed" / "2026-W21-summary.md"

190 + report_path = base / "diagnostics" / "preflight.json"

191 + raw_path.parent.mkdir(parents=True)

192 + output_path.parent.mkdir(parents=True)

193 + raw_path.write_text(

194 + json.dumps(

195 + {

196 + "week": "2026-W21",

197 + "new_repos": [{"full_name": "owner/new", "stars": 10}],

198 + "trending_repos": [{"full_name": "owner/trend", "stars": 20, "stars_gained": 5}],

199 + }

200 + ),

201 + encoding="utf-8",

202 + )

203 + prompt_template.write_text("{{RAW_JSON_CONTENT}}\n{{WISDOM}}\n{{SKILLS}}", encoding="utf-8")

204 +

205 + with mock.patch("sys.stdout", new_callable=io.StringIO) as stdout:

206 + exit_code = analyze_fallback.main(

207 + [

208 + "--raw-json",

209 + str(raw_path),

210 + "--output",

211 + str(output_path),

212 + "--current-datetime",

213 + "2026-05-18T13:05:53.678+02:00",

214 + "--prompt-template",

215 + str(prompt_template),

216 + "--analyzed-dir",

217 + str(output_path.parent),

218 + "--wisdom-file",

219 + str(base / "missing-wisdom.md"),

220 + "--skills-dir",

221 + str(base / "missing-skills"),

222 + "--preflight-report-json",

223 + str(report_path),

224 + "--print-prompt",

225 + ]

226 + )

227 +

228 + rendered = stdout.getvalue()

229 + report = json.loads(report_path.read_text(encoding="utf-8"))

230 + self.assertEqual(exit_code, 0)

231 + self.assertEqual(report["prompt_checksum_sha256"], analyze_fallback.checksum_text(rendered))

232 + self.assertEqual(report["deterministic_slices"], ["new_repos", "trending_repos", "press_correlations", "prior_continuity"])

233 + self.assertFalse(report["degraded"])

234 + self.assertTrue(report["publish_eligible"])

235 + self.assertEqual(report["promotion_policy"], "normal-promotion")

236 + self.assertIn("no-ai is diagnostic/staged-only", report["fallback_policy"])

237 + components = {component["name"]: component for component in report["components"]}

238 + self.assertEqual(components["new_repos"]["inclusion_reason"], "Deterministic mapper slice: newly discovered repositories.")

239 + self.assertEqual(components["trending_repos"]["compaction_decision"], "included")

240 +

241 + def test_preflight_compacts_before_prompt_exceeds_budget(self) -> None:

242 + tests_root = Path(__file__).resolve().parent

243 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

244 + base = Path(tmpdir)

245 + raw_path = base / "data" / "raw" / "2026-W21.json"

246 + prompt_template = base / "prompt.md"

247 + output_path = base / "data" / "analyzed" / "2026-W21-summary.md"

248 + report_path = base / "diagnostics" / "preflight.json"

249 + report_md_path = base / "diagnostics" / "preflight.md"

250 + raw_path.parent.mkdir(parents=True)

251 + output_path.parent.mkdir(parents=True)

252 + raw_path.write_text(

253 + json.dumps(

254 + {

255 + "week": "2026-W21",

256 + "new_repos": [{"full_name": f"owner/new-{i}", "stars": i} for i in range(60)],

257 + "trending_repos": [

258 + {"full_name": f"owner/trend-{i}", "stars": i, "stars_gained": i} for i in range(60)

259 + ],

260 + }

261 + ),

262 + encoding="utf-8",

263 + )

264 + prompt_template.write_text("{{RAW_JSON_CONTENT}}", encoding="utf-8")

265 +

266 + exit_code = analyze_fallback.main(

267 + [

268 + "--raw-json",

269 + str(raw_path),

270 + "--output",

271 + str(output_path),

272 + "--current-datetime",

273 + "2026-05-18T13:05:53.678+02:00",

274 + "--prompt-template",

275 + str(prompt_template),

276 + "--analyzed-dir",

277 + str(output_path.parent),

278 + "--preflight-report-json",

279 + str(report_path),

280 + "--preflight-report-md",

281 + str(report_md_path),

282 + "--prompt-token-budget",

283 + "2000",

284 + "--print-prompt",

285 + ]

286 + )

287 +

288 + report = json.loads(report_path.read_text(encoding="utf-8"))

289 + self.assertEqual(exit_code, 0)

290 + self.assertTrue(report["degraded"])

291 + self.assertFalse(report["publish_eligible"])

292 + self.assertIn("staged/candidate-only", report["promotion_policy"])

293 + self.assertIn("compacted", report["degradation_reason"])

294 + report_markdown = report_md_path.read_text(encoding="utf-8")

295 + self.assertIn("Degraded/compacted: `true`", report_markdown)

296 + self.assertIn("Publish eligible: `false`", report_markdown)

297 + self.assertIn("staged/candidate-only", report_markdown)

298 + components = {component["name"]: component for component in report["components"]}

299 + self.assertIn("compacted to top", components["new_repos"]["compaction_decision"])

300 + self.assertIn("compacted to top", components["trending_repos"]["compaction_decision"])

301 +

302 def test_extract_markdown_supports_message_parts(self) -> None:

303 payload = {

304 "choices": [

@@ -269,7 +388,7 @@ class AnalyzeFallbackTests(unittest.TestCase):

388 self.assertEqual(result.returncode, 0, result.stderr)

389 self.assertIn('week: "2026-W21"', result.stdout)

390

272 - def test_main_writes_fallback_output(self) -> None:

391 + def test_main_without_no_ai_rejects_github_models_fallback(self) -> None:

392 tests_root = Path(__file__).resolve().parent

393 with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

394 base = Path(tmpdir)

@@ -281,13 +400,9 @@ class AnalyzeFallbackTests(unittest.TestCase):

400 raw_path.write_text(json.dumps({"week": "2026-W21", "new_repos": [], "trending_repos": []}), encoding="utf-8")

401 prompt_template.write_text("{{RAW_JSON_CONTENT}}", encoding="utf-8")

402

284 - response = _FakeHTTPResponse(

285 - json.dumps({"choices": [{"message": {"content": "# Summary\n"}}]}).encode("utf-8")

286 - )

287 -

288 - with mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False), mock.patch.object(

289 - analyze_fallback.request, "urlopen", return_value=response

290 - ) as urlopen_mock:

403 + with mock.patch.object(analyze_fallback.request, "urlopen") as urlopen_mock, mock.patch(

404 + "sys.stderr", new_callable=io.StringIO

405 + ) as stderr:

406 exit_code = analyze_fallback.main(

407 [

408 "--raw-json",

@@ -303,9 +418,10 @@ class AnalyzeFallbackTests(unittest.TestCase):

418 ]

419 )

420

306 - self.assertEqual(exit_code, 0)

307 - self.assertEqual(output_path.read_text(encoding="utf-8"), "# Summary\n")

308 - self.assertEqual(urlopen_mock.call_args.kwargs["timeout"], analyze_fallback.DEFAULT_MODELS_TIMEOUT)

421 + self.assertEqual(exit_code, 1)

422 + self.assertFalse(output_path.exists())

423 + self.assertIn("GitHub Models/OpenAI analysis fallback is disabled", stderr.getvalue())

424 + urlopen_mock.assert_not_called()

425

426 def test_github_models_403_is_non_retryable_access_failure(self) -> None:

427 forbidden = error.HTTPError(

tests/test_copilot_failure.py

+2 -2

@@ -14,8 +14,8 @@ def test_classifies_copilot_token_failure_as_actionable_non_retryable() -> None:

14 assert report.retryable is False

15 assert report.actionable is True

16 assert "renew COPILOT_GH_TOKEN" in report.diagnostic

17 - assert copilot_failure.classify_log("HTTP 403 from Copilot").failure_class == "copilot_inaccessible"

18 - assert copilot_failure.classify_log("HTTP 401 from Copilot").failure_class == "copilot_inaccessible"

17 + assert copilot_failure.classify_log("HTTP 403 from Copilot").failure_class == "copilot_token_failure"

18 + assert copilot_failure.classify_log("HTTP 401 from Copilot").failure_class == "copilot_token_failure"

19 assert copilot_failure.classify_log("HTTP 403 invalid token").failure_class == "copilot_token_failure"

tests/test_pipeline.py

+27 -10

@@ -218,7 +218,7 @@ class WorkflowConfigTests(unittest.TestCase):

218

219 reskill_step = next((s for s in reskill["steps"] if s.get("name") == "Run reskill"), None)

220 self.assertIsNotNone(reskill_step)

221 - self.assertEqual(workflow["env"]["GITHUB_MODELS_MODEL"], "${{ vars.GITHUB_MODELS_MODEL || 'openai/gpt-4o' }}")

221 + self.assertNotIn("GITHUB_MODELS_MODEL", workflow["env"])

222 self.assertEqual(reskill_step["env"]["COPILOT_GITHUB_TOKEN"], "${{ secrets.COPILOT_GH_TOKEN }}")

223 reskill_run = reskill_step["run"]

224 self.assertIn("python3 scripts/reskill.py --current-datetime", reskill_run)

@@ -231,6 +231,10 @@ class WorkflowConfigTests(unittest.TestCase):

231 self.assertIn("trigger-log.txt", reskill_run)

232 self.assertIn("git add .squad/", reskill_run)

233 self.assertIn("data/metrics/", reskill_run)

234 + self.assertIn("no GitHub Models/OpenAI reskill fallback", reskill_run)

235 + self.assertNotIn("${GITHUB_MODELS_MODEL}", reskill_run)

236 + self.assertNotIn('RESKILL_SOURCE="github-models"', reskill_run)

237 + self.assertNotIn("used GitHub Models API fallback", reskill_run)

238 # Reskill prompt addresses the team, not an individual agent

239 self.assertIn('"Team, take a nap and reskill"', reskill_run)

240 self.assertNotIn("Farnsworth, read the file", reskill_run)

@@ -238,6 +242,17 @@ class WorkflowConfigTests(unittest.TestCase):

242 self.assertIn('RESKILL_PROMPT=".squad/reskill/current-prompt.md"', reskill_run)

243

244 analyze = workflow["jobs"]["analyze"]

245 + preflight_step = next((s for s in analyze["steps"] if s.get("name") == "Render and preflight analysis prompt"), None)

246 + self.assertIsNotNone(preflight_step)

247 + preflight_run = preflight_step["run"]

248 + self.assertIn("--prompt-token-budget", preflight_run)

249 + self.assertIn("--preflight-report-json", preflight_run)

250 + self.assertIn("--preflight-report-md", preflight_run)

251 + self.assertIn("--print-prompt > \"$PROMPT_FILE\"", preflight_run)

252 + self.assertIn("--context-files \"$PROMPT_FILE\"", preflight_run)

253 + self.assertIn("promotion_policy=", preflight_run)

254 + self.assertIn("staged/candidate-only", preflight_run)

255 +

256 run_analysis_step = next((s for s in analyze["steps"] if s.get("name") == "Run analysis"), None)

257 self.assertIsNotNone(run_analysis_step)

258 run_analysis = run_analysis_step["run"]

@@ -262,6 +277,12 @@ class WorkflowConfigTests(unittest.TestCase):

277 self.assertNotIn('ANALYSIS_SOURCE="github-models"', run_analysis)

278 self.assertNotIn("falling back to GitHub Models API", run_analysis)

279

280 + manifest_step = next((s for s in analyze["steps"] if s.get("name") == "Emit publish eligibility manifest"), None)

281 + self.assertIsNotNone(manifest_step)

282 + manifest_run = manifest_step["run"]

283 + self.assertEqual(manifest_step["env"]["PREFLIGHT_REPORT"], "${{ steps.prompt-preflight.outputs.preflight_report_json }}")

284 + self.assertIn('--preflight-report "$PREFLIGHT_REPORT"', manifest_run)

285 +

286 def test_generate_workflow_runs_rollups_and_commits_all_content(self) -> None:

287 workflow_path = Path(".github/workflows/crawl-and-publish.yml")

288 workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8"))

@@ -557,7 +578,7 @@ class PipelineIntegrationTests(unittest.TestCase):

578 self.assertNotIn("quality_score", rendered)

579 self.assertIn("## This Week's Trends", rendered)

580

560 - def test_analyze_fallback_can_process_raw_data(self) -> None:

581 + def test_analyze_fallback_no_ai_can_process_raw_data(self) -> None:

582 tests_root = Path(__file__).resolve().parent

583 with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

584 base = Path(tmpdir)

@@ -567,13 +588,7 @@ class PipelineIntegrationTests(unittest.TestCase):

588 output_path.parent.mkdir(parents=True)

589 raw_path.write_text(json.dumps(make_raw_payload()), encoding="utf-8")

590

570 - response = _FakeHTTPResponse(

571 - json.dumps({"choices": [{"message": {"content": make_analysis_markdown()}}]}).encode("utf-8")

572 - )

573 -

574 - with mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False), mock.patch.object(

575 - analyze_fallback.request, "urlopen", return_value=response

576 - ):

591 + with mock.patch.object(analyze_fallback.request, "urlopen") as urlopen_mock:

592 exit_code = analyze_fallback.main(

593 [

594 "--raw-json",

@@ -584,13 +599,15 @@ class PipelineIntegrationTests(unittest.TestCase):

599 FIXED_RUN_DATETIME,

600 "--analyzed-dir",

601 str(output_path.parent),

602 + "--no-ai",

603 ]

604 )

605

606 self.assertEqual(exit_code, 0)

607 written = output_path.read_text(encoding="utf-8")

592 - self.assertIn("Reliable Automation Gains Ground", written)

608 + self.assertIn("Automation, Observability, and This Week's Repo Signals", written)

609 self.assertIn("## Signal & Noise", written)

610 + urlopen_mock.assert_not_called()

611

612 def test_analysis_gate_validates_analysis_output_correctly(self) -> None:

613 tests_root = Path(__file__).resolve().parent

tests/test_promotion_guard.py

+55 -27

@@ -104,45 +104,73 @@ def write_gate_report(root: Path, path: Path, *, passed: bool = True) -> None:

104 )

105

106

107 +def write_preflight(root: Path, path: Path, *, degraded: bool = False, publish_eligible: bool = True) -> None:

108 + write_file(

109 + root,

110 + path.as_posix(),

111 + json.dumps(

112 + {

113 + "prompt_token_budget": 90000,

114 + "prompt_tokens": 1200,

115 + "prompt_bytes": 4800,

116 + "prompt_checksum_sha256": "a" * 64,

117 + "prompt_within_budget": True,

118 + "degraded": degraded,

119 + "publish_eligible": publish_eligible,

120 + "promotion_policy": "normal-promotion" if publish_eligible else "staged/candidate-only by default",

121 + "degradation_reason": "Prompt was deterministically compacted." if degraded else None,

122 + "fallback_policy": "copilot-only",

123 + "components": [],

124 + "deterministic_slices": [],

125 + }

126 + )

127 + + "\n",

128 + )

129 +

130 +

131 def create_publish_manifest(root: Path, name: str, *, source: str = "copilot-cli", model: str = "copilot-default", gate_passed: bool = True) -> Path:

132 candidate_dir = Path("data/candidates") / WEEK / name

133 summary_path = candidate_dir / f"{WEEK}-summary.md"

134 manifest_path = candidate_dir / "publish-manifest.json"

135 gate_report = candidate_dir / "analysis-gate-report.json"

136 + preflight_report = candidate_dir / "diagnostics" / "analysis-preflight.json"

137 write_file(root, summary_path.as_posix(), VALID_REPLACEMENT_SUMMARY)

138 write_publish_raw(root)

139 write_gate_report(root, gate_report, passed=gate_passed)

140 + if source == "copilot-cli":

141 + write_preflight(root, preflight_report)

142

143 previous_cwd = Path.cwd()

144 try:

145 os.chdir(root)

119 - publish_manifest.main(

120 - [

121 - "create",

122 - "--week",

123 - WEEK,

124 - "--run-id",

125 - name,

126 - "--current-datetime",

127 - RUN_STARTED_AT,

128 - "--summary",

129 - summary_path.as_posix(),

130 - "--published-summary",

131 - f"data/analyzed/{WEEK}-summary.md",

132 - "--raw-json",

133 - f"data/raw/{WEEK}.json",

134 - "--analysis-source",

135 - source,

136 - "--analysis-model",

137 - model,

138 - "--validation-status",

139 - "passed" if gate_passed else "failed",

140 - "--gate-report",

141 - gate_report.as_posix(),

142 - "--output",

143 - manifest_path.as_posix(),

144 - ]

145 - )

146 + args = [

147 + "create",

148 + "--week",

149 + WEEK,

150 + "--run-id",

151 + name,

152 + "--current-datetime",

153 + RUN_STARTED_AT,

154 + "--summary",

155 + summary_path.as_posix(),

156 + "--published-summary",

157 + f"data/analyzed/{WEEK}-summary.md",

158 + "--raw-json",

159 + f"data/raw/{WEEK}.json",

160 + "--analysis-source",

161 + source,

162 + "--analysis-model",

163 + model,

164 + "--validation-status",

165 + "passed" if gate_passed else "failed",

166 + "--gate-report",

167 + gate_report.as_posix(),

168 + "--output",

169 + manifest_path.as_posix(),

170 + ]

171 + if source == "copilot-cli":

172 + args.extend(["--preflight-report", preflight_report.as_posix()])

173 + publish_manifest.main(args)

174 finally:

175 os.chdir(previous_cwd)

176 return root / manifest_path

tests/test_publish_manifest.py

+126 -1

@@ -101,7 +101,45 @@ def write_gate_report(path: Path, *, passed: bool = True, errors: list[str] | No

101 )

102

103

104 -def create_args(base: Path, raw: Path, summary: Path, manifest: Path, *, source: str = "copilot-cli", model: str | None = "copilot-default", gate_report: Path | None = None, validation_status: str = "passed") -> list[str]:

104 +def write_preflight(path: Path, *, degraded: bool = False, publish_eligible: bool = True) -> None:

105 + path.parent.mkdir(parents=True, exist_ok=True)

106 + path.write_text(

107 + json.dumps(

108 + {

109 + "prompt_token_budget": 90000,

110 + "prompt_tokens": 1200,

111 + "prompt_bytes": 4800,

112 + "prompt_checksum_sha256": "a" * 64,

113 + "prompt_within_budget": True,

114 + "degraded": degraded,

115 + "publish_eligible": publish_eligible,

116 + "promotion_policy": (

117 + "normal-promotion"

118 + if not degraded

119 + else "staged/candidate-only by default; degraded compacted output requires an explicit future promotion policy."

120 + ),

121 + "degradation_reason": "Prompt was deterministically compacted." if degraded else None,

122 + "fallback_policy": "copilot-only",

123 + "components": [],

124 + "deterministic_slices": [],

125 + }

126 + ),

127 + encoding="utf-8",

128 + )

129 +

130 +

131 +def create_args(

132 + base: Path,

133 + raw: Path,

134 + summary: Path,

135 + manifest: Path,

136 + *,

137 + source: str = "copilot-cli",

138 + model: str | None = "copilot-default",

139 + gate_report: Path | None = None,

140 + validation_status: str = "passed",

141 + preflight: Path | None | bool = True,

142 +) -> list[str]:

143 args = [

144 "create",

145 "--week",

@@ -127,6 +165,12 @@ def create_args(base: Path, raw: Path, summary: Path, manifest: Path, *, source:

165 args.extend(["--analysis-model", model])

166 if gate_report is not None:

167 args.extend(["--gate-report", str(gate_report)])

168 + if preflight is True and source == "copilot-cli":

169 + preflight_path = manifest.parent / "diagnostics" / "analysis-preflight.json"

170 + write_preflight(preflight_path)

171 + args.extend(["--preflight-report", str(preflight_path)])

172 + elif isinstance(preflight, Path):

173 + args.extend(["--preflight-report", str(preflight)])

174 return args

175

176

@@ -160,6 +204,9 @@ class PublishManifestTests(unittest.TestCase):

204 payload = json.loads(manifest.read_text(encoding="utf-8"))

205 self.assertEqual(payload["schema_version"], "publish_eligibility_v1")

206 self.assertEqual(payload["analysis"]["ai_status"], "ai")

207 + self.assertEqual(payload["analysis"]["preflight"]["degraded"], False)

208 + self.assertEqual(payload["analysis"]["preflight"]["publish_eligible"], True)

209 + self.assertEqual(payload["analysis"]["preflight"]["promotion_policy"], "normal-promotion")

210 self.assertTrue(payload["promotion"]["eligible"])

211 self.assertEqual(payload["promotion"]["decision"], "promote")

212 self.assertRegex(payload["candidate"]["summary_sha256"], r"^[0-9a-f]{64}$")

@@ -192,6 +239,84 @@ class PublishManifestTests(unittest.TestCase):

239 with self.assertRaises(SystemExit):

240 assert_eligible_from_root(base, manifest)

241

242 + def test_copilot_ai_candidate_requires_preflight_for_promotion(self) -> None:

243 + tests_root = Path(__file__).resolve().parent

244 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

245 + base = Path(tmpdir)

246 + raw = base / "data/raw/2026-W21.json"

247 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"

248 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"

249 + gate_report = base / "data/candidates/2026-W21/123456/analysis-gate-report.json"

250 + write_raw(raw)

251 + write_summary(summary)

252 + write_gate_report(gate_report)

253 +

254 + publish_manifest.main(

255 + create_args(base, raw, summary, manifest, gate_report=gate_report, preflight=False)

256 + )

257 +

258 + payload = json.loads(manifest.read_text(encoding="utf-8"))

259 + self.assertEqual(payload["analysis"]["ai_status"], "ai")

260 + self.assertFalse(payload["promotion"]["eligible"])

261 + self.assertTrue(any("preflight report is required" in reason for reason in payload["promotion"]["reasons"]))

262 +

263 + def test_github_models_source_is_not_ai_publishable(self) -> None:

264 + tests_root = Path(__file__).resolve().parent

265 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

266 + base = Path(tmpdir)

267 + raw = base / "data/raw/2026-W21.json"

268 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"

269 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"

270 + gate_report = base / "data/candidates/2026-W21/123456/analysis-gate-report.json"

271 + write_raw(raw)

272 + write_summary(summary)

273 + write_gate_report(gate_report)

274 +

275 + publish_manifest.main(

276 + create_args(

277 + base,

278 + raw,

279 + summary,

280 + manifest,

281 + source="github-models",

282 + model="openai/gpt-4o",

283 + gate_report=gate_report,

284 + )

285 + )

286 +

287 + payload = json.loads(manifest.read_text(encoding="utf-8"))

288 + self.assertEqual(payload["analysis"]["ai_status"], "unknown")

289 + self.assertFalse(payload["promotion"]["eligible"])

290 + self.assertTrue(any("analysis source is not AI-publishable" in reason for reason in payload["promotion"]["reasons"]))

291 +

292 + def test_degraded_preflight_candidate_is_staged_only_by_default(self) -> None:

293 + tests_root = Path(__file__).resolve().parent

294 + with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:

295 + base = Path(tmpdir)

296 + raw = base / "data/raw/2026-W21.json"

297 + summary = base / "data/candidates/2026-W21/123456/2026-W21-summary.md"

298 + manifest = base / "data/candidates/2026-W21/123456/publish-manifest.json"

299 + gate_report = base / "data/candidates/2026-W21/123456/analysis-gate-report.json"

300 + preflight = base / "data/candidates/2026-W21/123456/diagnostics/analysis-preflight.json"

301 + write_raw(raw)

302 + write_summary(summary)

303 + write_gate_report(gate_report)

304 + write_preflight(preflight, degraded=True, publish_eligible=False)

305 +

306 + publish_manifest.main(

307 + create_args(base, raw, summary, manifest, gate_report=gate_report, preflight=preflight)

308 + )

309 +

310 + payload = json.loads(manifest.read_text(encoding="utf-8"))

311 + self.assertFalse(payload["promotion"]["eligible"])

312 + self.assertEqual(payload["promotion"]["decision"], "block")

313 + self.assertTrue(payload["analysis"]["preflight"]["degraded"])

314 + self.assertFalse(payload["analysis"]["preflight"]["publish_eligible"])

315 + self.assertIn("staged/candidate-only", payload["analysis"]["preflight"]["promotion_policy"])

316 + self.assertTrue(any("preflight degraded/compacted" in reason for reason in payload["promotion"]["reasons"]))

317 + with self.assertRaises(SystemExit):

318 + assert_eligible_from_root(base, manifest)

319 +

320 def test_copilot_candidate_without_explicit_model_uses_publishable_default(self) -> None:

321 tests_root = Path(__file__).resolve().parent

322 with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir: