fix: prevent copilot stdout leaks in weekly outputs (#146)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 20, 2026 at 22:21 UTC
baaf5c57eceefa077168f7c7366faa14f3f16ed5
6 files changed
+140
-3
.github/workflows/crawl-and-publish.yml
+16
-2
@@ -294,6 +294,10 @@ jobs:
294
PROMPT_FILE=$(mktemp)
295
python3 scripts/analyze_fallback.py --raw-json "$WEEK_FILE" --output "$OUTPUT_FILE" --current-datetime "$CURRENT_DATETIME" --print-prompt > "$PROMPT_FILE"
296
297
+ sanitize_agent_output() {
298
+ python3 scripts/sanitize_agent_output.py --path "$1"
299
+ }
300
+
301
# Append press context to prompt if available
302
if [ -f "$PRESS_FILE" ] && [ -s "$PRESS_FILE" ]; then
303
printf '\n\n---\n## Press Context\n\n' >> "$PROMPT_FILE"
@@ -311,7 +315,7 @@ jobs:
315
--allow-tool=glob \
316
--allow-tool=grep \
317
--share=data/metrics/copilot-transcript.md \
314
- > "$OUTPUT_FILE"; then
318
+ > /dev/null; then
319
ANALYSIS_SOURCE="copilot-cli"
320
ANALYSIS_MODEL="copilot-default"
321
elif python3 scripts/analyze_fallback.py \
@@ -334,6 +338,8 @@ jobs:
338
ANALYSIS_MODEL="none"
339
fi
340
341
+ sanitize_agent_output "$OUTPUT_FILE"
342
+
343
TRANSCRIPT_ARGS=""
344
if [ -f "data/metrics/copilot-transcript.md" ]; then
345
TRANSCRIPT_ARGS="--transcript data/metrics/copilot-transcript.md"
@@ -737,6 +743,10 @@ jobs:
743
RESKILL_PROMPT=$(mktemp)
744
mkdir -p .squad/skills .squad/reskill data/metrics
745
746
+ sanitize_agent_output() {
747
+ python3 scripts/sanitize_agent_output.py --path "$1"
748
+ }
749
+
750
# Primary path: Copilot CLI with agent identity
751
RESKILL_SOURCE="copilot-cli"
752
RESKILL_MODEL="copilot-default"
@@ -752,7 +762,7 @@ jobs:
762
--allow-tool=write \
763
--allow-tool=glob \
764
--allow-tool=grep \
755
- > "$RESKILL_OUTPUT"; then
765
+ > /dev/null; then
766
echo "✅ Reskill via Copilot CLI with agent identity"
767
# Fallback: GitHub Models API via reskill.py
768
elif python3 scripts/reskill.py --current-datetime "$CURRENT_DATETIME" --output "$RESKILL_OUTPUT" --prompt-output "$RESKILL_PROMPT"; then
@@ -767,6 +777,10 @@ jobs:
777
echo "Reskill triggered at run #$COUNTER ($CURRENT_DATETIME)" >> .squad/reskill/trigger-log.txt
778
fi
779
780
+ if [ "$RESKILL_SOURCE" != "none" ]; then
781
+ sanitize_agent_output "$RESKILL_OUTPUT"
782
+ fi
783
+
784
if [ "$RESKILL_SOURCE" != "none" ]; then
785
API_RESPONSE_ARGS=""
786
if [ -f "data/metrics/reskill-api-response.json" ]; then
.squad/agents/farnsworth/history.md
+1
@@ -32,3 +32,4 @@
32
- **2026-05-19T20:57:55Z:** PR #139 merged. Correlation Summary section now renders as narrative prose in reader_mode (Decision recorded in `.squad/decisions.md`). Groups correlations by organization, ranks by aggregate confidence, fetches README snippets for top 2 repos per group (max 6 total), produces 1–3 interpretive paragraphs with inline links. Graceful failure on README timeout (5s). Key rationale: raw repo names with scores communicate data but not meaning; narrative format helps readers understand organizational impact. AI-mode output unchanged. Additionally: reskill job now catches RuntimeError on model 403, emits placeholder report, exits cleanly. Related decisions: `_format_correlations_narrative()` function pattern, README fetch timeout strategy, segment-by-org grouping logic. All new code covered by tests; 513 total pass.
33
- **2026-05-20T19:15:53.942+02:00:** Article structure restructured from repo-listing format to Gartner/McKinsey-style trend brief (requested by jmservera). New section order: `## This Week's Trends` → `## Where Industry Meets Code` → `## Signal & Noise` → `## Blind Spots` → `## The Week Ahead` → `## Key References` (with `### Notable Projects` and `### Press & Industry`). Key design decisions: (1) Lead with named macro trends — 3-5 synthesized themes, not repo lists. (2) "Where Industry Meets Code" replaces "Industry & Press Correlation" — editorial framing, not ML-sounding. (3) Signal & Noise merged into integrated prose section (no `### Signal`/`### Noise` sub-headings) for better editorial writing. (4) Repos move to `### Notable Projects` at end as references, freeing body prose to cite them as evidence rather than leading with them. (5) `generate_rollups.py` updated with backward compat — tries new heading names first, falls back to old for existing `2026-W21-summary.md`. Files changed: `prompts/analyze-weekly.md`, `docs/analysis-spec.md`, `scripts/analysis_gate.py`, `scripts/analyze_fallback.py`, `scripts/generate_rollups.py`, 5 test files. All 519 tests pass. Decision filed at `.squad/decisions/inbox/farnsworth-article-restructure.md`.
34
- **2026-05-20T20:09:26+02:00:** Removed stale `claude-sonnet-4` pins from `crawl-and-publish.yml` so analysis and reskill now rely on the Copilot CLI default model instead of hardcoded model IDs. Added a `copilot-default` cost profile for preflight/usage tracking and made `GITHUB_MODELS_MODEL` workflow-wide and configurable via repo vars with `openai/gpt-4o` as the fallback default. Lesson: for platform-managed Copilot CLI runs, pinning a versioned model name is operational debt; keep the CLI on its moving default and only pin fallback API models you explicitly control.
35
+- **2026-05-20T22:14:02+02:00:** Fixed a publication leak where Copilot CLI stdout was redirected into the same markdown file that Farnsworth writes via the `write` tool. Analysis and reskill now discard CLI stdout, then run an explicit post-step sanitizer over the output file to strip any leaked agent epilogues such as `✅ Farnsworth is done`, `Editorial thesis:`, or `Quality score:` lines. Lesson: when an agent writes a file directly, shell stdout is a separate channel and must never target the same path; add a cleanup pass anyway because transcript-style metadata can still escape through unexpected CLI behavior.
.squad/decisions/inbox/farnsworth-stdout-leak-fix.md
new
+13
@@ -0,0 +1,13 @@
1
+# 2026-05-20: Prevent Copilot stdout from leaking into published markdown
2
+
3
+- **Owner:** Farnsworth
4
+- **Date:** 2026-05-20T22:14:02+02:00
5
+- **Status:** Proposed
6
+- **Decision:** In `crawl-and-publish.yml`, Copilot CLI stdout must never be redirected to the same markdown file that the agent writes via the `write` tool. Analysis and reskill invocations should send stdout to `/dev/null`, rely on `--share` or workflow logs for transcripts, and run a post-write sanitizer over the target markdown file as defense in depth.
7
+- **Why:** The published week 21 article leaked agent status text because the shell appended Copilot CLI stdout to `data/analyzed/2026-W21-summary.md` after Farnsworth had already written the real article. The same collision pattern existed in the reskill path. Separating channels fixes the root cause, and a sanitizer reduces blast radius if the CLI emits metadata again.
8
+- **Implementation notes:**
9
+ - Changed both Copilot CLI redirects in `.github/workflows/crawl-and-publish.yml` from the output markdown file to `/dev/null`.
10
+ - Added `scripts/sanitize_agent_output.py` and invoked it after analysis/reskill generation to strip leaked lines such as `✅ Farnsworth is done`, `Editorial thesis:`, and `Quality score:`.
11
+ - Reinforced `prompts/analyze-weekly.md` so the agent writes only publication-ready markdown beginning with YAML frontmatter and ending with the final article line.
12
+ - Left the quality gate reading `$OUTPUT_FILE`; it now validates the agent-written markdown only.
13
+- **Scope:** `.github/workflows/crawl-and-publish.yml`, `prompts/analyze-weekly.md`, `scripts/sanitize_agent_output.py`, `tests/test_sanitize_agent_output.py`
prompts/analyze-weekly.md
+4
-1
@@ -46,6 +46,8 @@ The analyze job must resolve both learned-state placeholders before invoking Cop
46
47
Write the full contents of `{{OUTPUT_PATH}}` as markdown with YAML frontmatter. The file must conform to the Output Contract in `docs/analysis-spec.md` exactly.
48
49
+The output file is publication-ready content, not a chat transcript. Do not include status updates, self-evaluation, tool notes, agent identity markers, quality explanations, or any other meta-commentary before, after, or inside the article.
50
+
51
## Editorial stance
52
53
Be critical, selective, and opinionated.
@@ -115,9 +117,10 @@ Be critical, selective, and opinionated.
117
- `## The Week Ahead`: ~50-110 words. Forward-looking editorial close. What should readers watch for next week? What trends are in motion that haven't peaked yet? Where is the ecosystem heading based on this week's evidence?
118
- `## Key References` with `### Notable Projects` (5-10 most important repos with 1-sentence context each) and `### Press & Industry` (3-5 most relevant articles or sources, or "No press data was provided this week." if absent).
119
15. The body must be at least 200 words.
118
-16. Do not include raw JSON, notes to self, placeholders, or tool transcripts.
120
+16. Do not include raw JSON, notes to self, placeholders, tool transcripts, status summaries, self-referential text, or quality-score commentary outside the required frontmatter fields.
121
17. Every repository reference in the body must be a clickable GitHub markdown link in this exact format: `[owner/repo](https://github.com/owner/repo)`.
122
18. Output only the finished markdown file content.
123
+19. The first characters in the file must be the opening `---` of the YAML frontmatter, and the file must end after the final article line with no agent epilogue.
124
125
## Working method
126
scripts/sanitize_agent_output.py
new
+58
@@ -0,0 +1,58 @@
1
+#!/usr/bin/env python3
2
+from __future__ import annotations
3
+
4
+import argparse
5
+import re
6
+from pathlib import Path
7
+
8
+META_LINE_PATTERNS = [
9
+ re.compile(r"^✅\s+.+\bis done\..*$"),
10
+ re.compile(r"^Quality score:\s+.*$", re.IGNORECASE),
11
+ re.compile(r"^Editorial thesis:\s+.*$", re.IGNORECASE),
12
+ re.compile(r"^data/analyzed/.+\.md is written.*$", re.IGNORECASE),
13
+ re.compile(r"^\.squad/reskill/.+\.md is written.*$", re.IGNORECASE),
14
+]
15
+
16
+
17
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
18
+ parser = argparse.ArgumentParser(description="Strip leaked Copilot/Farnsworth meta lines from markdown outputs.")
19
+ parser.add_argument("--path", required=True, type=Path, help="File to sanitize in place.")
20
+ return parser.parse_args(argv)
21
+
22
+
23
+def sanitize_text(text: str) -> str:
24
+ sanitized_lines: list[str] = []
25
+ changed = False
26
+ for line in text.splitlines():
27
+ stripped = line.strip()
28
+ if stripped and any(pattern.match(stripped) for pattern in META_LINE_PATTERNS):
29
+ changed = True
30
+ continue
31
+ sanitized_lines.append(line)
32
+ sanitized = "\n".join(sanitized_lines)
33
+ if text.endswith("\n"):
34
+ sanitized += "\n"
35
+ if changed:
36
+ sanitized = re.sub(r"\n{3,}", "\n\n", sanitized)
37
+ return sanitized
38
+
39
+
40
+def sanitize_file(path: Path) -> bool:
41
+ if not path.exists() or not path.is_file():
42
+ return False
43
+ original = path.read_text(encoding="utf-8")
44
+ sanitized = sanitize_text(original)
45
+ if sanitized == original:
46
+ return False
47
+ path.write_text(sanitized, encoding="utf-8")
48
+ return True
49
+
50
+
51
+def main(argv: list[str] | None = None) -> int:
52
+ args = parse_args(argv)
53
+ sanitize_file(args.path)
54
+ return 0
55
+
56
+
57
+if __name__ == "__main__":
58
+ raise SystemExit(main())
tests/test_sanitize_agent_output.py
new
+48
@@ -0,0 +1,48 @@
1
+import unittest
2
+
3
+import scripts.sanitize_agent_output as sanitize_agent_output
4
+
5
+
6
+class SanitizeAgentOutputTests(unittest.TestCase):
7
+ def test_sanitize_text_removes_leaked_agent_summary_lines(self) -> None:
8
+ original = """---
9
+title: \"Week 21, 2026 Analysis\"
10
+quality_score: 76
11
+---
12
+
13
+## This Week's Trends
14
+
15
+Real article paragraph.
16
+
17
+✅ Farnsworth is done. data/analyzed/2026-W21-summary.md is written (88 lines).
18
+Editorial thesis: Genuine agentic coding infrastructure is breaking out.
19
+Quality score: 76/100 — publishable.
20
+"""
21
+
22
+ sanitized = sanitize_agent_output.sanitize_text(original)
23
+
24
+ self.assertNotIn("✅ Farnsworth is done.", sanitized)
25
+ self.assertNotIn("Editorial thesis:", sanitized)
26
+ self.assertNotIn("Quality score: 76/100", sanitized)
27
+ self.assertIn("Real article paragraph.", sanitized)
28
+ self.assertIn("quality_score: 76", sanitized)
29
+
30
+ def test_sanitize_text_preserves_legitimate_article_content(self) -> None:
31
+ original = """---
32
+title: \"Week 21, 2026 Analysis\"
33
+quality_score: 76
34
+summary: \"Quality score caveats belong only in frontmatter.\"
35
+---
36
+
37
+## Signal & Noise
38
+
39
+The article discusses quality score inflation as a concept without using agent metadata.
40
+"""
41
+
42
+ sanitized = sanitize_agent_output.sanitize_text(original)
43
+
44
+ self.assertEqual(sanitized, original)
45
+
46
+
47
+if __name__ == "__main__":
48
+ unittest.main()