feat: narrative Correlation Summary in reader_mode (#138)
Replace the raw confidence/match_type bullet list in the Correlation Summary section with flowing prose paragraphs when reader_mode=True. - Add _fetch_readme_snippet(): fetches first 500 chars of a repo README from raw.githubusercontent.com (5s timeout, fails gracefully) - Add _extract_readme_description(): extracts first readable sentence from README snippet, stripping markdown/HTML formatting - Add _format_correlations_narrative(): groups correlations by org, fetches README snippets for top repos (max 6 total), and generates 1-3 paragraphs with inline links to repos and matched TC articles - Modify format_correlations_list(): add reader_mode and articles params; delegates to _format_correlations_narrative when reader_mode=True - Update render_press_context(): passes reader_mode + articles to format_correlations_list - Update tests: replace bullet-list reader_mode checks with narrative assertions; add TestExtractReadmeDescription and TestFormatCorrelationsNarrative test classes (14 new tests, 513 total) README fetching only happens in reader_mode=True paths; AI-mode output (reader_mode=False) is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 19, 2026 at 22:42 UTC
f8500751ce6f1f29f55b081bffeeccd98b3f7573
2 files changed
+313
-24
scripts/render_press_context.py
+192
-5
@@ -10,7 +10,9 @@ Usage:
10
11
import argparse
12
import json
13
+import re
14
import sys
15
+import urllib.request
16
from datetime import datetime
17
from pathlib import Path
18
@@ -56,17 +58,196 @@ def format_articles_list(articles: list[dict]) -> str:
58
_HYPE_RISK_SEVERITY: dict[str, int] = {"high": 3, "medium": 2, "low": 1, "none": 0}
59
60
59
-def format_correlations_list(correlations: list[dict], *, top_n: int | None = None) -> str:
60
- """Format correlations into a markdown list.
61
+def _fetch_readme_snippet(full_name: str, max_chars: int = 500) -> str:
62
+ """Fetch the first max_chars of a repo README from raw.githubusercontent.com.
63
+
64
+ Returns an empty string on any failure (network error, 404, timeout).
65
+ Should only be called in reader_mode=True paths.
66
+ """
67
+ url = f"https://raw.githubusercontent.com/{full_name}/HEAD/README.md"
68
+ try:
69
+ req = urllib.request.Request(url, headers={"User-Agent": "SquadScope/1.0"})
70
+ with urllib.request.urlopen(req, timeout=5) as resp:
71
+ raw = resp.read(max_chars * 3)
72
+ return raw.decode("utf-8", errors="replace")[:max_chars]
73
+ except Exception:
74
+ return ""
75
+
76
+
77
+def _extract_readme_description(snippet: str) -> str:
78
+ """Return the first readable descriptive line from a README snippet.
79
+
80
+ Skips headings, badge lines, image tags, and blank lines.
81
+ Returns an empty string if nothing usable is found.
82
+ """
83
+ for line in snippet.splitlines():
84
+ line = line.strip()
85
+ if not line:
86
+ continue
87
+ if line.startswith(("#", "!", "<", "|", "[")):
88
+ continue
89
+ # Strip markdown formatting: links → text, remove bold/italic/code, strip HTML
90
+ line = re.sub(r"!\[.*?\]\(.*?\)", "", line)
91
+ line = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", line)
92
+ line = re.sub(r"<[^>]+>", "", line)
93
+ line = re.sub(r"[*_`>]", "", line)
94
+ line = line.strip(" .,;:")
95
+ if 20 <= len(line) <= 150:
96
+ return line
97
+ return ""
98
+
99
+
100
+def _format_correlations_narrative(
101
+ correlations: list[dict], articles: list[dict]
102
+) -> str:
103
+ """Generate narrative paragraphs explaining press-to-code correlations.
104
+
105
+ Groups correlations by GitHub org, fetches README snippets for top repos,
106
+ and produces 1–3 prose paragraphs with inline links to repos and articles.
107
+ Only called in reader_mode=True — README network fetches happen here.
108
+ """
109
+ if not correlations:
110
+ return "(No significant press correlations this week.)"
111
+
112
+ # URL → title lookup for inline article links
113
+ url_to_title: dict[str, str] = {
114
+ a["url"]: a["title"]
115
+ for a in articles
116
+ if a.get("url") and a.get("title")
117
+ }
118
+
119
+ # Sort correlations by confidence desc, hype_risk severity desc
120
+ sorted_corrs = sorted(
121
+ correlations,
122
+ key=lambda c: (
123
+ -c.get("correlation_confidence", 0.0),
124
+ -_HYPE_RISK_SEVERITY.get(c.get("hype_risk", "none"), 0),
125
+ ),
126
+ )
127
+
128
+ # Group by org (first segment of "owner/repo")
129
+ org_groups: dict[str, list[dict]] = {}
130
+ for corr in sorted_corrs:
131
+ repo = corr.get("repo", "")
132
+ if not repo:
133
+ continue
134
+ org = repo.split("/")[0]
135
+ org_groups.setdefault(org, []).append(corr)
136
+
137
+ def _group_score(corrs: list[dict]) -> float:
138
+ return sum(c.get("correlation_confidence", 0.0) for c in corrs)
139
+
140
+ top_groups = sorted(
141
+ org_groups.items(),
142
+ key=lambda kv: _group_score(kv[1]),
143
+ reverse=True,
144
+ )[:4]
145
+
146
+ # Fetch README snippets for the top repos across groups (max 6 total)
147
+ repos_to_fetch: list[str] = []
148
+ for _, group_corrs in top_groups:
149
+ for corr in group_corrs[:2]:
150
+ repo = corr.get("repo", "")
151
+ if repo and repo not in repos_to_fetch and len(repos_to_fetch) < 6:
152
+ repos_to_fetch.append(repo)
153
+
154
+ readme_snippets: dict[str, str] = {}
155
+ for repo in repos_to_fetch:
156
+ snippet = _fetch_readme_snippet(repo)
157
+ if snippet:
158
+ readme_snippets[repo] = snippet
159
+
160
+ total = len(correlations)
161
+ paragraphs: list[str] = []
162
+
163
+ for idx, (org, group_corrs) in enumerate(top_groups[:3]):
164
+ # Collect up to 2 article links for this group
165
+ article_links: list[str] = []
166
+ seen_article_urls: set[str] = set()
167
+ for corr in group_corrs:
168
+ for url in corr.get("matched_articles", []):
169
+ if url not in seen_article_urls and len(article_links) < 2:
170
+ seen_article_urls.add(url)
171
+ title = url_to_title.get(url, "")
172
+ if title:
173
+ article_links.append(f"[{title}]({url})")
174
+
175
+ # Collect up to 3 repo links with optional README description
176
+ repo_parts: list[str] = []
177
+ for corr in group_corrs[:3]:
178
+ repo = corr.get("repo", "")
179
+ if not repo:
180
+ continue
181
+ link = _repo_link(repo)
182
+ desc = _extract_readme_description(readme_snippets.get(repo, ""))
183
+ repo_parts.append(f"{link} — {desc}" if desc else link)
184
+
185
+ if not repo_parts:
186
+ continue
187
+
188
+ repos_str = _join_links(repo_parts)
189
+
190
+ if article_links:
191
+ arts_str = _join_links(article_links)
192
+ if idx == 0:
193
+ para = (
194
+ f"This week's TechCrunch coverage closely tracks developer activity "
195
+ f"across {total} repos. {org.capitalize()} featured prominently: "
196
+ f"coverage of {arts_str} aligns with activity in {repos_str}."
197
+ )
198
+ else:
199
+ para = (
200
+ f"{org.capitalize()}'s press footprint also intersects with GitHub: "
201
+ f"coverage of {arts_str} tracks activity in {repos_str}."
202
+ )
203
+ else:
204
+ if idx == 0:
205
+ para = (
206
+ f"This week's TechCrunch coverage closely tracks developer activity "
207
+ f"across {total} repos. {org.capitalize()} shows the strongest signal, "
208
+ f"with {repos_str} seeing notable GitHub traction."
209
+ )
210
+ else:
211
+ para = (
212
+ f"{org.capitalize()} also shows strong press-to-code correlation, "
213
+ f"with activity in {repos_str}."
214
+ )
215
+
216
+ paragraphs.append(para)
217
+
218
+ return (
219
+ "\n\n".join(paragraphs)
220
+ if paragraphs
221
+ else "(No significant press correlations this week.)"
222
+ )
223
+
224
+
225
+def format_correlations_list(
226
+ correlations: list[dict],
227
+ *,
228
+ top_n: int | None = None,
229
+ reader_mode: bool = False,
230
+ articles: list[dict] | None = None,
231
+) -> str:
232
+ """Format correlations into a markdown list or narrative prose.
233
234
Args:
235
correlations: List of correlation dicts.
64
- top_n: When set, show only the top N entries (sorted by confidence desc,
65
- then hype_risk severity desc) and append a "…and N more" summary line.
236
+ top_n: When set (and reader_mode=False), show only the top N entries
237
+ (sorted by confidence desc, then hype_risk severity desc) and
238
+ append a "…and N more" summary line.
239
+ reader_mode: When True, delegate to _format_correlations_narrative()
240
+ which produces prose paragraphs with inline links. top_n
241
+ is ignored in this mode.
242
+ articles: Article list used by the narrative formatter for URL→title
243
+ lookup. Ignored when reader_mode=False.
244
"""
245
if not correlations:
246
return "- (none)"
247
248
+ if reader_mode:
249
+ return _format_correlations_narrative(correlations, articles or [])
250
+
251
if top_n is not None:
252
sorted_corrs = sorted(
253
correlations,
@@ -351,7 +532,13 @@ def render_press_context(
532
rendered = rendered.replace("{articles_list}", format_articles_list(articles))
533
rendered = rendered.replace("{correlation_count}", str(correlation_count))
534
rendered = rendered.replace(
354
- "{correlations_list}", format_correlations_list(correlations, top_n=top_n)
535
+ "{correlations_list}",
536
+ format_correlations_list(
537
+ correlations,
538
+ top_n=top_n,
539
+ reader_mode=reader_mode,
540
+ articles=articles,
541
+ ),
542
)
543
544
# Strip the AI-only ### Instructions block in reader mode
tests/test_render_press_context.py
+121
-19
@@ -7,6 +7,9 @@ _REPO_ROOT = Path(__file__).resolve().parent.parent
7
sys.path.insert(0, str(_REPO_ROOT / "scripts"))
8
9
from render_press_context import (
10
+ _extract_readme_description,
11
+ _fetch_readme_snippet,
12
+ _format_correlations_narrative,
13
format_articles_list,
14
format_correlations_list,
15
format_divergences,
@@ -305,33 +308,33 @@ class TestRenderPressContextReaderMode:
308
assert "### Instructions" in result
309
assert "Press-correlated" in result
310
308
- def test_reader_mode_truncates_large_correlations(self):
311
+ def test_reader_mode_uses_narrative(self):
312
many = [
313
{
311
- "repo": f"org/repo-{i}",
312
- "match_type": "keyword",
313
- "correlation_confidence": 0.5,
314
- "hype_risk": "low",
314
+ "repo": f"openai/repo-{i}",
315
+ "match_type": "org_name",
316
+ "correlation_confidence": 0.8,
317
+ "hype_risk": "medium",
318
+ "matched_articles": ["https://techcrunch.com/article"],
319
}
320
for i in range(20)
321
]
318
- result = render_press_context(
319
- _techcrunch_data(),
320
- _correlation_data(many),
321
- "2026-W21",
322
- reader_mode=True,
323
- )
324
- repo_lines = [ln for ln in result.splitlines() if ln.startswith("- org/repo")]
325
- assert len(repo_lines) == 10
326
- assert "…and 10 more repos with press correlation" in result
327
-
328
- def test_reader_mode_no_truncation_when_under_limit(self):
322
+ tc = _techcrunch_data([_article(title="OpenAI Launch", url="https://techcrunch.com/article")])
323
+ result = render_press_context(tc, _correlation_data(many), "2026-W21", reader_mode=True)
324
+ # Narrative mode: no raw confidence/match_type bullets
325
+ assert "confidence:" not in result
326
+ assert "match_type" not in result
327
+ # Should contain prose with repo links
328
+ assert "https://github.com/openai/repo-" in result
329
+
330
+ def test_reader_mode_no_raw_confidence_in_narrative(self):
331
few = [
332
{
331
- "repo": f"org/repo-{i}",
332
- "match_type": "keyword",
333
- "correlation_confidence": 0.5,
333
+ "repo": f"google/repo-{i}",
334
+ "match_type": "org_name",
335
+ "correlation_confidence": 0.9,
336
"hype_risk": "low",
337
+ "matched_articles": [],
338
}
339
for i in range(5)
340
]
@@ -341,7 +344,9 @@ class TestRenderPressContextReaderMode:
344
"2026-W21",
345
reader_mode=True,
346
)
347
+ assert "confidence:" not in result
348
assert "more repos with press correlation" not in result
349
+ assert "https://github.com/google/repo-" in result
350
351
352
class TestStripAiInstructions:
@@ -414,3 +419,100 @@ class TestStripAiInstructions:
419
result = self.af._strip_ai_instructions(content)
420
assert "more repos with press correlation" not in result
421
assert result.count("- org/repo") == 5
422
+
423
+
424
+class TestExtractReadmeDescription:
425
+ def test_returns_first_readable_line(self):
426
+ snippet = "# My Project\n\nA fast, zero-dependency library for data processing.\n"
427
+ assert _extract_readme_description(snippet) == "A fast, zero-dependency library for data processing"
428
+
429
+ def test_skips_heading_lines(self):
430
+ snippet = "# Heading\n## Subheading\nActual description here.\n"
431
+ assert _extract_readme_description(snippet) == "Actual description here"
432
+
433
+ def test_skips_image_badge_lines(self):
434
+ snippet = "[](url)\nA concise description of what this library does.\n"
435
+ assert _extract_readme_description(snippet) == "A concise description of what this library does"
436
+
437
+ def test_returns_empty_on_no_match(self):
438
+ assert _extract_readme_description("# Only a heading\n") == ""
439
+
440
+ def test_strips_markdown_links(self):
441
+ snippet = "Check out [our docs](https://example.com) for more information.\n"
442
+ result = _extract_readme_description(snippet)
443
+ assert "https://example.com" not in result
444
+ assert "our docs" in result
445
+
446
+
447
+class TestFormatCorrelationsNarrative:
448
+ def _corr(self, repo="openai/codex", confidence=0.8, hype_risk="medium", articles=None):
449
+ return {
450
+ "repo": repo,
451
+ "press_correlated": True,
452
+ "correlation_confidence": confidence,
453
+ "matched_articles": articles or [],
454
+ "match_type": "org_name",
455
+ "hype_risk": hype_risk,
456
+ }
457
+
458
+ def _art(self, title="OpenAI News", url="https://techcrunch.com/openai-news"):
459
+ return {"title": title, "url": url, "categories": ["AI"]}
460
+
461
+ def test_empty_returns_fallback(self):
462
+ result = _format_correlations_narrative([], [])
463
+ assert "No significant press correlations" in result
464
+
465
+ def test_produces_repo_links(self):
466
+ corr = self._corr(repo="openai/codex", articles=["https://techcrunch.com/a1"])
467
+ result = _format_correlations_narrative([corr], [self._art(url="https://techcrunch.com/a1")])
468
+ assert "[codex](https://github.com/openai/codex)" in result
469
+
470
+ def test_produces_article_links_when_title_available(self):
471
+ corr = self._corr(articles=["https://techcrunch.com/a1"])
472
+ art = self._art(title="OpenAI Launches Codex", url="https://techcrunch.com/a1")
473
+ result = _format_correlations_narrative([corr], [art])
474
+ assert "[OpenAI Launches Codex](https://techcrunch.com/a1)" in result
475
+
476
+ def test_no_raw_confidence_in_output(self):
477
+ corr = self._corr()
478
+ result = _format_correlations_narrative([corr], [])
479
+ assert "confidence:" not in result
480
+ assert "match_type" not in result
481
+ assert "hype_risk" not in result
482
+
483
+ def test_groups_by_org(self):
484
+ corrs = [
485
+ self._corr(repo="openai/codex"),
486
+ self._corr(repo="openai/gpt-4"),
487
+ self._corr(repo="google/material-design-icons", confidence=0.5),
488
+ ]
489
+ result = _format_correlations_narrative(corrs, [])
490
+ # openai dominates — should appear in first paragraph
491
+ assert "openai" in result.lower()
492
+ assert "google" in result.lower()
493
+
494
+ def test_no_article_link_when_url_not_in_articles(self):
495
+ corr = self._corr(articles=["https://techcrunch.com/unknown-url"])
496
+ result = _format_correlations_narrative([corr], [])
497
+ # URL is in the corr but not in the articles list, so no link text
498
+ assert "[" not in result or "github.com" in result
499
+
500
+ def test_reader_mode_true_uses_narrative(self):
501
+ corr = self._corr(repo="openai/codex")
502
+ result = format_correlations_list([corr], reader_mode=True, articles=[])
503
+ assert "confidence:" not in result
504
+ assert "[codex](https://github.com/openai/codex)" in result
505
+
506
+ def test_reader_mode_false_uses_bullet_list(self):
507
+ corr = self._corr(repo="openai/codex")
508
+ result = format_correlations_list([corr], reader_mode=False)
509
+ assert "openai/codex" in result
510
+ assert "confidence:" in result
511
+ assert "match:" in result
512
+
513
+ def test_ai_mode_unchanged_in_render(self):
514
+ tc = _techcrunch_data()
515
+ corr_data = _correlation_data([self._corr()])
516
+ result = render_press_context(tc, corr_data, "2026-W21", reader_mode=False)
517
+ assert "confidence:" in result
518
+ assert "### Instructions" in result