fix: make weekly analysis Copilot-only
Remove GitHub Models fallback assumptions and add Copilot failure classification/token-renewal issue handling.
Juan Manuel Servera committed
Jun 6, 2026 at 10:04 UTC
8119130d55e580996f82ad3807bd0e888d8b54df
7 files changed
+415
-55
.github/workflows/crawl-and-publish.yml
+39
-41
@@ -200,6 +200,7 @@ jobs:
200
permissions:
201
actions: read
202
contents: write
203
+ issues: write
204
outputs:
205
week: ${{ steps.analysis-context.outputs.week }}
206
summary_file: ${{ steps.analysis-context.outputs.published_output_file }}
@@ -406,6 +407,7 @@ jobs:
407
ANALYSIS_SOURCE=""
408
ANALYSIS_MODEL=""
409
LAST_GATE_FINGERPRINT=""
410
+ FINAL_FAILURE_CLASS=""
411
412
if command -v copilot >/dev/null 2>&1; then
413
while [ "$GATE_PASSED" = "false" ] && [ "$ATTEMPT" -le "$MAX_RETRIES" ]; do
@@ -420,7 +422,9 @@ jobs:
422
TRANSCRIPT_FILE="data/metrics/copilot-transcript-attempt-${ATTEMPT}.md"
423
GATE_REPORT="$DIAGNOSTICS_DIR/gate-copilot-cli-attempt-${ATTEMPT}.json"
424
CANDIDATE_SNAPSHOT="$DIAGNOSTICS_DIR/candidate-copilot-cli-attempt-${ATTEMPT}.md"
423
- rm -f "$TRANSCRIPT_FILE"
425
+ COPILOT_LOG="$DIAGNOSTICS_DIR/copilot-cli-attempt-${ATTEMPT}.log"
426
+ COPILOT_FAILURE_REPORT="$DIAGNOSTICS_DIR/copilot-cli-failure-attempt-${ATTEMPT}.json"
427
+ rm -f "$TRANSCRIPT_FILE" "$COPILOT_LOG" "$COPILOT_FAILURE_REPORT"
428
REPAIR_CONTEXT=""
429
if [ "$ATTEMPT" -gt 0 ]; then
430
PREVIOUS_REPORT="$DIAGNOSTICS_DIR/gate-copilot-cli-attempt-$((ATTEMPT - 1)).json"
@@ -430,7 +434,8 @@ jobs:
434
fi
435
436
# Intentionally rely on Copilot CLI's default model so CI follows the platform-supported default.
433
- if ! copilot \
437
+ set +e
438
+ copilot \
439
--agent squad \
440
-p "Farnsworth, read the file at ${PROMPT_FILE} — it contains the weekly data and analysis instructions. Follow them exactly and write the analysis to ${OUTPUT_FILE}.${REPAIR_CONTEXT}" \
441
-s \
@@ -440,12 +445,30 @@ jobs:
445
--allow-tool=glob \
446
--allow-tool=grep \
447
--share="$TRANSCRIPT_FILE" \
443
- > /dev/null; then
444
- echo "::warning::Copilot CLI failed on attempt $((ATTEMPT + 1))"
448
+ > "$COPILOT_LOG" 2>&1
449
+ COPILOT_STATUS=$?
450
+ set -e
451
+ if [ "$COPILOT_STATUS" -ne 0 ]; then
452
+ FAILURE_CLASS=$(python3 scripts/copilot_failure.py \
453
+ --log "$COPILOT_LOG" \
454
+ --exit-code "$COPILOT_STATUS" \
455
+ --report-json "$COPILOT_FAILURE_REPORT" \
456
+ --create-token-issue \
457
+ --repo "${GITHUB_REPOSITORY:-jmservera/SquadScope}" \
458
+ --assignee "jmservera" \
459
+ --week "$WEEK" \
460
+ --run-id "${GITHUB_RUN_ID:-local}")
461
+ FINAL_FAILURE_CLASS="$FAILURE_CLASS"
462
+ echo "::warning::Copilot CLI failed on attempt $((ATTEMPT + 1)); class=${FAILURE_CLASS}; report=${COPILOT_FAILURE_REPORT}"
463
+ if [ "$FAILURE_CLASS" = "copilot_token_failure" ] || [ "$FAILURE_CLASS" = "copilot_inaccessible" ] || [ "$FAILURE_CLASS" = "context_too_large" ]; then
464
+ echo "::error::Non-retryable Copilot analysis failure (${FAILURE_CLASS}). See ${COPILOT_FAILURE_REPORT}."
465
+ exit 1
466
+ fi
467
ATTEMPT=$((ATTEMPT + 1))
468
continue
469
fi
470
471
+ FINAL_FAILURE_CLASS=""
472
sanitize_agent_output "$OUTPUT_FILE"
473
474
# Inline quality gate check (suppress step summary to avoid noise).
@@ -473,49 +496,24 @@ jobs:
496
ATTEMPT=$((ATTEMPT + 1))
497
done
498
else
476
- echo "::warning::Copilot CLI unavailable; falling back to GitHub Models API."
499
+ FINAL_FAILURE_CLASS="copilot_inaccessible"
500
+ echo "::error::Copilot CLI unavailable; this repository has no GitHub Models/OpenAI fallback."
501
+ exit 1
502
fi
503
504
if [ "$GATE_PASSED" = "false" ]; then
480
- echo "::warning::No publishable Copilot summary was produced; falling back to GitHub Models API."
481
- MODELS_PASSED="false"
482
- MODELS_GATE_REPORT="$DIAGNOSTICS_DIR/gate-github-models-attempt-0.json"
483
- if python3 scripts/analyze_fallback.py \
505
+ echo "::error::No publishable Copilot summary was produced; this repository has no GitHub Models/OpenAI fallback. Final failure class: ${FINAL_FAILURE_CLASS:-quality_gate}"
506
+ python3 scripts/analyze_fallback.py \
507
--raw-json "$WEEK_FILE" \
508
--output "$OUTPUT_FILE" \
509
--current-datetime "$CURRENT_DATETIME" \
487
- --press-context "$PRESS_FILE"; then
488
- sanitize_agent_output "$OUTPUT_FILE"
489
- if run_quality_gate github-models "$MODELS_GATE_REPORT"; then
490
- cp "$OUTPUT_FILE" "$DIAGNOSTICS_DIR/candidate-github-models-attempt-0.md" 2>/dev/null || true
491
- MODELS_PASSED="true"
492
- ANALYSIS_SOURCE="github-models"
493
- ANALYSIS_MODEL="${GITHUB_MODELS_MODEL:-openai/gpt-4o}"
494
- else
495
- cp "$OUTPUT_FILE" "$DIAGNOSTICS_DIR/candidate-github-models-attempt-0.md" 2>/dev/null || true
496
- echo "::warning::GitHub Models output failed quality gate; falling back to data-only no-AI summary."
497
- fi
498
- else
499
- echo "::warning::GitHub Models fallback failed; falling back to data-only no-AI summary."
500
- fi
501
-
502
- if [ "$MODELS_PASSED" = "false" ]; then
503
- python3 scripts/analyze_fallback.py \
504
- --raw-json "$WEEK_FILE" \
505
- --output "$OUTPUT_FILE" \
506
- --current-datetime "$CURRENT_DATETIME" \
507
- --press-context "$PRESS_FILE" \
508
- --no-ai
509
- sanitize_agent_output "$OUTPUT_FILE"
510
- NO_AI_GATE_REPORT="$DIAGNOSTICS_DIR/gate-no-ai-attempt-0.json"
511
- if ! run_quality_gate no-ai "$NO_AI_GATE_REPORT"; then
512
- echo "::error::Analysis quality gate failed for Copilot CLI, GitHub Models API, and no-AI outputs."
513
- exit 1
514
- fi
515
- cp "$OUTPUT_FILE" "$DIAGNOSTICS_DIR/candidate-no-ai-attempt-0.md" 2>/dev/null || true
516
- ANALYSIS_SOURCE="no-ai"
517
- ANALYSIS_MODEL="none"
518
- fi
510
+ --press-context "$PRESS_FILE" \
511
+ --no-ai
512
+ sanitize_agent_output "$OUTPUT_FILE"
513
+ NO_AI_GATE_REPORT="$DIAGNOSTICS_DIR/gate-no-ai-attempt-0.json"
514
+ run_quality_gate no-ai "$NO_AI_GATE_REPORT" || true
515
+ cp "$OUTPUT_FILE" "$DIAGNOSTICS_DIR/candidate-no-ai-attempt-0.md" 2>/dev/null || true
516
+ exit 1
517
fi
518
519
# Copy final transcript to canonical location
docs/operator-guide.md
+2
-2
@@ -231,7 +231,7 @@ Navigate to **Actions → Crawl and Publish** in your repo. Green checkmarks = s
231
4. Update the secret: `gh secret set COPILOT_GH_TOKEN --body YOUR_NEW_PAT -R YOUR_USERNAME/SquadScope`
232
5. Re-run the workflow
233
234
-**Fallback:** The workflow automatically falls back to GitHub Models API if Copilot fails. Check the workflow logs to see which path was used.
234
+**Fallback:** There is no GitHub Models/OpenAI fallback for weekly analysis. Token/auth failures fail immediately and create or update an issue assigned to `@jmservera` to renew `COPILOT_GH_TOKEN`.
235
236
### ❌ GitHub API rate limits exceeded
237
@@ -255,7 +255,7 @@ Navigate to **Actions → Crawl and Publish** in your repo. Green checkmarks = s
255
**Fix:**
256
1. Check your Copilot usage: https://github.com/settings/copilot
257
2. If you hit the limit, wait for the next billing cycle or upgrade your plan
258
-3. The workflow will automatically fall back to GitHub Models API (lower quality, but functional)
258
+3. The workflow retries transient Copilot failures, but no GitHub Models/OpenAI fallback is configured for weekly analysis.
259
260
**Mitigation:** Copilot Pro includes generous quota. For automated pipelines, consider Copilot Team (more quota, better for organizations).
261
docs/pipeline-validation.md
+6
-6
@@ -14,7 +14,7 @@ This checklist validates the automated weekly workflow in `.github/workflows/cra
14
Required secrets/tokens:
15
16
- `COPILOT_GH_TOKEN` — fine-grained PAT used as `COPILOT_GITHUB_TOKEN` for Copilot CLI analysis.
17
-- `GITHUB_TOKEN` — built-in workflow token used for crawling, artifact downloads, commits, fallback GitHub Models calls, and Pages deployment.
17
+- `GITHUB_TOKEN` — built-in workflow token used for crawling, artifact downloads, commits, token-renewal issue creation, and Pages deployment.
18
19
## Stage-by-stage validation
20
@@ -51,7 +51,7 @@ Required secrets/tokens:
51
**Inputs**
52
- `raw-data` artifact downloaded into `data/raw/`
53
- `COPILOT_GH_TOKEN` for Copilot CLI primary path
54
-- `GITHUB_TOKEN` for GitHub Models fallback
54
+- `GITHUB_TOKEN` for diagnostics, commits, and Copilot-token renewal issue creation
55
56
**Outputs**
57
- `data/analyzed/YYYY-WNN-summary.md`
@@ -65,9 +65,9 @@ Required secrets/tokens:
65
- Current raw file week matches the run week
66
- Correlation and press-context steps consume compact external-news data with legacy `YYYY-WNN-techcrunch.json` fallback
67
- Press context preserves source names, article URLs/titles/dates, strong-vs-weak labels, and partial-source caveats while staying under the ~8k token budget
68
-- Copilot CLI output or fallback output is written to `data/analyzed/`
68
+- Copilot CLI output is written to `data/analyzed/`; if Copilot cannot produce publishable analysis, no-AI output is diagnostic only and the run fails.
69
- `scripts/analysis_gate.py` passes before publish continues
70
-- Job permissions include `actions: read`, `contents: write`, `copilot-requests: write`, and `models: read`
70
+- Job permissions include `actions: read`, `contents: write`, and `issues: write`
71
72
### 3. Generate
73
@@ -129,14 +129,14 @@ Required secrets/tokens:
129
- External news crawl: `python3 -m scripts.techcrunch_crawler --sources config/external_news_sources.json --output data/raw/YYYY-WNN-external-news.json --since YYYY-MM-DD --until YYYY-MM-DD`
130
- Correlate press: `python3 -m scripts.correlate --raw data/raw/YYYY-WNN.json --techcrunch data/raw/YYYY-WNN-external-news.json --output data/analyzed/YYYY-WNN-correlations.json`
131
- Render press context: `python3 -m scripts.render_press_context --week YYYY-WNN`
132
-- Analyze fallback: `python3 scripts/analyze_fallback.py --raw-json data/raw/YYYY-WNN.json --output data/analyzed/YYYY-WNN-summary.md --current-datetime YYYY-MM-DDTHH:MM:SSZ`
132
+- Render analysis prompt/diagnostic no-AI output: `python3 scripts/analyze_fallback.py --raw-json data/raw/YYYY-WNN.json --output data/analyzed/YYYY-WNN-summary.md --current-datetime YYYY-MM-DDTHH:MM:SSZ --print-prompt`
133
- Gate: `python3 scripts/analysis_gate.py --analysis-file data/analyzed/YYYY-WNN-summary.md --raw-json data/raw/YYYY-WNN.json --current-datetime YYYY-MM-DDTHH:MM:SSZ`
134
- Generate: `python3 scripts/generate_content.py data/analyzed/YYYY-WNN-summary.md`
135
- Deploy build check: `hugo --minify`
136
137
## Known limitations and workarounds
138
139
-- Copilot CLI in CI depends on `COPILOT_GH_TOKEN`; when unavailable, the workflow falls back to GitHub Models automatically.
139
+- Copilot CLI in CI depends on `COPILOT_GH_TOKEN`; when token/auth fails, the workflow fails immediately and creates or updates an issue assigned to `@jmservera` to renew the token. There is no GitHub Models/OpenAI fallback for weekly analysis.
140
- Weekly momentum quality is only as good as the historical star snapshots; first runs and sparse history can make `stars_gained` incomplete.
141
- Hugo must be `0.146.0+`; the workflow pins `0.161.1` because older runner binaries fail with the current theme.
142
- The scheduled workflow now deploys Pages directly. `deploy-site.yml` skips bot-authored pushes so the scheduled run does not trigger a duplicate Pages deployment.
prompts/analyze-weekly.md
+1
-1
@@ -33,7 +33,7 @@ Use this only if it is provided. If it is missing, unavailable, or empty, say so
33
34
## Learned context
35
36
-The analyze job must resolve both learned-state placeholders before invoking Copilot CLI or the GitHub Models fallback.
36
+The analyze job must resolve both learned-state placeholders before invoking Copilot CLI. Weekly AI analysis is Copilot-only; there is no GitHub Models/OpenAI fallback configured for this repository.
37
38
1. Read `.squad/identity/wisdom.md` and inject its current contents into `{{WISDOM}}`.
39
2. Read markdown files under `.squad/skills/` (for example `SKILL.md` files in nested skill folders), concatenate them in a stable sorted order, and inject that bundle into `{{SKILLS}}`.
scripts/copilot_failure.py
new
+235
@@ -0,0 +1,235 @@
1
+#!/usr/bin/env python3
2
+from __future__ import annotations
3
+
4
+import argparse
5
+import json
6
+import subprocess
7
+from dataclasses import asdict, dataclass
8
+from pathlib import Path
9
+
10
+
11
+@dataclass
12
+class CopilotFailure:
13
+ failure_class: str
14
+ retryable: bool
15
+ actionable: bool
16
+ diagnostic: str
17
+ exit_code: int | None = None
18
+
19
+
20
+TOKEN_PATTERNS = (
21
+ "bad credentials",
22
+ "invalid token",
23
+ "expired token",
24
+ "token expired",
25
+ "authentication failed",
26
+ "unauthorized",
27
+ "copilot_github_token",
28
+)
29
+INACCESSIBLE_PATTERNS = (
30
+ "401",
31
+ "403",
32
+ "copilot is not available",
33
+ "copilot unavailable",
34
+ "permission",
35
+ "not subscribed",
36
+ "subscription",
37
+ "access to copilot",
38
+ "forbidden",
39
+)
40
+CONTEXT_PATTERNS = (
41
+ "context length",
42
+ "context too large",
43
+ "maximum context",
44
+ "token limit",
45
+ "too many tokens",
46
+ "prompt is too long",
47
+)
48
+TIMEOUT_PATTERNS = ("timed out", "timeout", "deadline exceeded")
49
+TRANSIENT_PATTERNS = (
50
+ "rate limit",
51
+ "429",
52
+ "500",
53
+ "502",
54
+ "503",
55
+ "504",
56
+ "econnreset",
57
+ "network",
58
+ "temporarily unavailable",
59
+ "temporary failure",
60
+)
61
+
62
+
63
+def classify_log(log_text: str, exit_code: int | None = None) -> CopilotFailure:
64
+ normalized = log_text.lower()
65
+ if any(pattern in normalized for pattern in TOKEN_PATTERNS):
66
+ return CopilotFailure(
67
+ "copilot_token_failure",
68
+ retryable=False,
69
+ actionable=True,
70
+ diagnostic="Copilot authentication/token failure; renew COPILOT_GH_TOKEN.",
71
+ exit_code=exit_code,
72
+ )
73
+ if any(pattern in normalized for pattern in CONTEXT_PATTERNS):
74
+ return CopilotFailure(
75
+ "context_too_large",
76
+ retryable=False,
77
+ actionable=True,
78
+ diagnostic="Copilot prompt/context exceeded supported size; reduce analysis context before rerun.",
79
+ exit_code=exit_code,
80
+ )
81
+ if any(pattern in normalized for pattern in TIMEOUT_PATTERNS):
82
+ return CopilotFailure(
83
+ "timeout",
84
+ retryable=True,
85
+ actionable=False,
86
+ diagnostic="Copilot analysis timed out; retry is allowed.",
87
+ exit_code=exit_code,
88
+ )
89
+ if any(pattern in normalized for pattern in TRANSIENT_PATTERNS):
90
+ return CopilotFailure(
91
+ "transient_error",
92
+ retryable=True,
93
+ actionable=False,
94
+ diagnostic="Copilot analysis hit a transient service/network failure; retry is allowed.",
95
+ exit_code=exit_code,
96
+ )
97
+ if any(pattern in normalized for pattern in INACCESSIBLE_PATTERNS):
98
+ return CopilotFailure(
99
+ "copilot_inaccessible",
100
+ retryable=False,
101
+ actionable=True,
102
+ diagnostic="Copilot is inaccessible for this workflow; verify Copilot availability and permissions.",
103
+ exit_code=exit_code,
104
+ )
105
+ return CopilotFailure(
106
+ "other",
107
+ retryable=True,
108
+ actionable=False,
109
+ diagnostic="Copilot analysis failed with an unclassified error; retry is allowed before failing the run.",
110
+ exit_code=exit_code,
111
+ )
112
+
113
+
114
+def issue_title() -> str:
115
+ return "Renew GitHub Copilot token for weekly analysis workflow"
116
+
117
+
118
+def issue_body(report: CopilotFailure, *, week: str, run_id: str) -> str:
119
+ return "\n".join(
120
+ [
121
+ "The weekly analysis workflow cannot run because GitHub Copilot authentication failed.",
122
+ "",
123
+ f"- Week: `{week}`",
124
+ f"- Run ID: `{run_id}`",
125
+ f"- Failure class: `{report.failure_class}`",
126
+ f"- Diagnostic: {report.diagnostic}",
127
+ "",
128
+ "Please renew or replace the `COPILOT_GH_TOKEN` secret, then rerun the workflow.",
129
+ ]
130
+ )
131
+
132
+
133
+def run_gh(args: list[str]) -> subprocess.CompletedProcess[str]:
134
+ return subprocess.run(["gh", *args], check=False, capture_output=True, text=True)
135
+
136
+
137
+def issue_url(repo: str, number: str) -> str:
138
+ return f"https://github.com/{repo}/issues/{number}"
139
+
140
+
141
+def create_or_update_token_issue(report: CopilotFailure, *, repo: str, assignee: str, week: str, run_id: str) -> str:
142
+ title = issue_title()
143
+ body = issue_body(report, week=week, run_id=run_id)
144
+ search = run_gh(
145
+ [
146
+ "issue",
147
+ "list",
148
+ "--repo",
149
+ repo,
150
+ "--state",
151
+ "open",
152
+ "--search",
153
+ title,
154
+ "--json",
155
+ "number,title",
156
+ "--limit",
157
+ "10",
158
+ ]
159
+ )
160
+ if search.returncode == 0:
161
+ try:
162
+ issues = json.loads(search.stdout)
163
+ except json.JSONDecodeError:
164
+ issues = []
165
+ for issue in issues:
166
+ if issue.get("title") == title and issue.get("number"):
167
+ number = str(issue["number"])
168
+ comment = run_gh(["issue", "comment", number, "--repo", repo, "--body", body])
169
+ if comment.returncode != 0:
170
+ raise RuntimeError(comment.stderr.strip() or "failed to update Copilot token issue")
171
+ return issue_url(repo, number)
172
+
173
+ created = run_gh(
174
+ [
175
+ "issue",
176
+ "create",
177
+ "--repo",
178
+ repo,
179
+ "--title",
180
+ title,
181
+ "--body",
182
+ body,
183
+ "--assignee",
184
+ assignee,
185
+ "--label",
186
+ "type:bug",
187
+ ]
188
+ )
189
+ if created.returncode != 0:
190
+ raise RuntimeError(created.stderr.strip() or "failed to create Copilot token issue")
191
+ created_output = created.stdout.strip()
192
+ if created_output.startswith("https://"):
193
+ return created_output
194
+ return created_output or issue_url(repo, "unknown")
195
+
196
+
197
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
198
+ parser = argparse.ArgumentParser(description="Classify Copilot CLI analysis failures.")
199
+ parser.add_argument("--log", required=True, type=Path)
200
+ parser.add_argument("--exit-code", type=int)
201
+ parser.add_argument("--report-json", type=Path)
202
+ parser.add_argument("--create-token-issue", action="store_true")
203
+ parser.add_argument("--repo", default="")
204
+ parser.add_argument("--assignee", default="jmservera")
205
+ parser.add_argument("--week", default="")
206
+ parser.add_argument("--run-id", default="")
207
+ return parser.parse_args(argv)
208
+
209
+
210
+def main(argv: list[str] | None = None) -> int:
211
+ args = parse_args(argv)
212
+ log_text = args.log.read_text(encoding="utf-8", errors="replace") if args.log.exists() else ""
213
+ report = classify_log(log_text, args.exit_code)
214
+ payload = asdict(report)
215
+ payload["log_path"] = args.log.as_posix()
216
+
217
+ if args.create_token_issue and report.failure_class == "copilot_token_failure":
218
+ payload["issue"] = create_or_update_token_issue(
219
+ report,
220
+ repo=args.repo,
221
+ assignee=args.assignee,
222
+ week=args.week,
223
+ run_id=args.run_id,
224
+ )
225
+
226
+ if args.report_json:
227
+ args.report_json.parent.mkdir(parents=True, exist_ok=True)
228
+ args.report_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
229
+
230
+ print(report.failure_class)
231
+ return 0
232
+
233
+
234
+if __name__ == "__main__":
235
+ raise SystemExit(main())
tests/test_copilot_failure.py
new
+125
@@ -0,0 +1,125 @@
1
+from __future__ import annotations
2
+
3
+import json
4
+from pathlib import Path
5
+from unittest import mock
6
+
7
+from scripts import copilot_failure
8
+
9
+
10
+def test_classifies_copilot_token_failure_as_actionable_non_retryable() -> None:
11
+ report = copilot_failure.classify_log("Authentication failed: Bad credentials", exit_code=1)
12
+
13
+ assert report.failure_class == "copilot_token_failure"
14
+ assert report.retryable is False
15
+ assert report.actionable is True
16
+ assert "renew COPILOT_GH_TOKEN" in report.diagnostic
17
+ assert copilot_failure.classify_log("HTTP 403 from Copilot").failure_class == "copilot_inaccessible"
18
+ assert copilot_failure.classify_log("HTTP 401 from Copilot").failure_class == "copilot_inaccessible"
19
+ assert copilot_failure.classify_log("HTTP 403 invalid token").failure_class == "copilot_token_failure"
20
+
21
+
22
+def test_classifies_context_timeout_and_transient_failures() -> None:
23
+ assert copilot_failure.classify_log("maximum context length exceeded").failure_class == "context_too_large"
24
+ assert copilot_failure.classify_log("request timed out").failure_class == "timeout"
25
+ assert copilot_failure.classify_log("HTTP 503 temporarily unavailable").failure_class == "transient_error"
26
+
27
+
28
+def test_main_writes_report_without_creating_issue_for_transient(tmp_path: Path) -> None:
29
+ log_path = tmp_path / "copilot.log"
30
+ report_path = tmp_path / "report.json"
31
+ log_path.write_text("HTTP 429 rate limit", encoding="utf-8")
32
+
33
+ with mock.patch.object(copilot_failure, "create_or_update_token_issue") as issue_mock:
34
+ exit_code = copilot_failure.main(
35
+ [
36
+ "--log",
37
+ str(log_path),
38
+ "--exit-code",
39
+ "1",
40
+ "--report-json",
41
+ str(report_path),
42
+ "--create-token-issue",
43
+ "--repo",
44
+ "jmservera/SquadScope",
45
+ ]
46
+ )
47
+
48
+ payload = json.loads(report_path.read_text(encoding="utf-8"))
49
+ assert exit_code == 0
50
+ assert payload["failure_class"] == "transient_error"
51
+ assert payload["retryable"] is True
52
+ issue_mock.assert_not_called()
53
+
54
+
55
+def test_main_creates_or_updates_issue_for_token_failure(tmp_path: Path) -> None:
56
+ log_path = tmp_path / "copilot.log"
57
+ report_path = tmp_path / "report.json"
58
+ log_path.write_text("COPILOT_GITHUB_TOKEN invalid token", encoding="utf-8")
59
+
60
+ with mock.patch.object(
61
+ copilot_failure,
62
+ "create_or_update_token_issue",
63
+ return_value="https://github.com/jmservera/SquadScope/issues/123",
64
+ ) as issue_mock:
65
+ exit_code = copilot_failure.main(
66
+ [
67
+ "--log",
68
+ str(log_path),
69
+ "--exit-code",
70
+ "1",
71
+ "--report-json",
72
+ str(report_path),
73
+ "--create-token-issue",
74
+ "--repo",
75
+ "jmservera/SquadScope",
76
+ "--week",
77
+ "2026-W23",
78
+ "--run-id",
79
+ "27055543722",
80
+ ]
81
+ )
82
+
83
+ payload = json.loads(report_path.read_text(encoding="utf-8"))
84
+ assert exit_code == 0
85
+ assert payload["failure_class"] == "copilot_token_failure"
86
+ assert payload["issue"] == "https://github.com/jmservera/SquadScope/issues/123"
87
+ issue_mock.assert_called_once()
88
+
89
+
90
+def test_create_or_update_token_issue_returns_consistent_url_for_existing_issue() -> None:
91
+ report = copilot_failure.classify_log("invalid token")
92
+ responses = [
93
+ mock.Mock(returncode=0, stdout='[{"number": 123, "title": "Renew GitHub Copilot token for weekly analysis workflow"}]', stderr=""),
94
+ mock.Mock(returncode=0, stdout="", stderr=""),
95
+ ]
96
+
97
+ with mock.patch.object(copilot_failure, "run_gh", side_effect=responses):
98
+ result = copilot_failure.create_or_update_token_issue(
99
+ report,
100
+ repo="jmservera/SquadScope",
101
+ assignee="jmservera",
102
+ week="2026-W23",
103
+ run_id="27055543722",
104
+ )
105
+
106
+ assert result == "https://github.com/jmservera/SquadScope/issues/123"
107
+
108
+
109
+def test_create_or_update_token_issue_returns_consistent_url_for_created_issue() -> None:
110
+ report = copilot_failure.classify_log("invalid token")
111
+ responses = [
112
+ mock.Mock(returncode=0, stdout="[]", stderr=""),
113
+ mock.Mock(returncode=0, stdout="https://github.com/jmservera/SquadScope/issues/124\n", stderr=""),
114
+ ]
115
+
116
+ with mock.patch.object(copilot_failure, "run_gh", side_effect=responses):
117
+ result = copilot_failure.create_or_update_token_issue(
118
+ report,
119
+ repo="jmservera/SquadScope",
120
+ assignee="jmservera",
121
+ week="2026-W23",
122
+ run_id="27055543722",
123
+ )
124
+
125
+ assert result == "https://github.com/jmservera/SquadScope/issues/124"
tests/test_pipeline.py
+7
-5
@@ -245,15 +245,17 @@ class WorkflowConfigTests(unittest.TestCase):
245
self.assertNotIn("--model claude-sonnet-4", run_analysis)
246
self.assertIn("mkdir -p data/metrics", run_analysis)
247
self.assertIn("run_quality_gate()", run_analysis)
248
- self.assertIn("falling back to GitHub Models API", run_analysis)
248
+ self.assertIn("python3 scripts/copilot_failure.py", run_analysis)
249
+ self.assertIn("--create-token-issue", run_analysis)
250
+ self.assertIn('FINAL_FAILURE_CLASS=""', run_analysis)
251
self.assertIn("No publishable Copilot summary was produced", run_analysis)
252
+ self.assertIn("no GitHub Models/OpenAI fallback", run_analysis)
253
self.assertIn("python3 scripts/analyze_fallback.py", run_analysis)
254
self.assertIn('--press-context "$PRESS_FILE"', run_analysis)
252
- self.assertIn('ANALYSIS_SOURCE="github-models"', run_analysis)
255
self.assertIn("--no-ai", run_analysis)
254
- self.assertIn('ANALYSIS_SOURCE="no-ai"', run_analysis)
255
- self.assertIn('ANALYSIS_MODEL="none"', run_analysis)
256
- self.assertIn("GitHub Models fallback failed; falling back to data-only no-AI summary", run_analysis)
256
+ self.assertIn("exit 1", run_analysis)
257
+ self.assertNotIn('ANALYSIS_SOURCE="github-models"', run_analysis)
258
+ self.assertNotIn("falling back to GitHub Models API", run_analysis)
259
260
def test_generate_workflow_runs_rollups_and_commits_all_content(self) -> None:
261
workflow_path = Path(".github/workflows/crawl-and-publish.yml")