Generate monthly and yearly rollups with append-only updates (#42)
* feat: generate monthly and yearly rollups Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Integrate Pagefind search and taxonomy generation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix rollup review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <copilot@github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 18, 2026 at 16:17 UTC
f0b1dee227323af6353dbe7689441b430ae9506b
3 files changed
+157
-24
scripts/generate_rollups.py
+55
-13
@@ -14,8 +14,11 @@ if __package__ in {None, ""}:
14
15
import scripts.analysis_gate as analysis_gate
16
17
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
18
SUMMARY_SUFFIX = "-summary.md"
19
WEEK_PATTERN = re.compile(r"^(?P<year>\d{4})-W(?P<week>\d{2})$")
20
+REPO_LINK_PATTERN = re.compile(r"https://github\.com/(?P<repo>[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)")
21
+NO_UPDATES_PLACEHOLDER = "_No updates yet._"
22
MONTHLY_SECTIONS = [
23
"Month Overview",
24
"Top Repos This Month",
@@ -62,6 +65,7 @@ class WeeklySummary:
65
tags: tuple[str, ...]
66
repos_featured: int
67
top_repo: str
68
+ featured_repos: tuple[str, ...]
69
summary: str
70
signal: str
71
noise: str
@@ -111,9 +115,21 @@ class RollupPage:
115
116
117
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
114
- parser = argparse.ArgumentParser(description="Generate append-only monthly and yearly rollups from weekly analyses.")
115
- parser.add_argument("--analyzed-dir", type=Path, default=Path("data/analyzed"), help="Directory containing weekly summary markdown files.")
116
- parser.add_argument("--content-root", type=Path, default=Path("content"), help="Root content directory for generated rollups.")
118
+ parser = argparse.ArgumentParser(
119
+ description="Generate append-only monthly and yearly rollups from weekly analyses. Defaults resolve from the repository root."
120
+ )
121
+ parser.add_argument(
122
+ "--analyzed-dir",
123
+ type=Path,
124
+ default=PROJECT_ROOT / "data" / "analyzed",
125
+ help="Directory containing weekly summary markdown files.",
126
+ )
127
+ parser.add_argument(
128
+ "--content-root",
129
+ type=Path,
130
+ default=PROJECT_ROOT / "content",
131
+ help="Root content directory for generated rollups.",
132
+ )
133
return parser.parse_args(argv)
134
135
@@ -138,7 +154,7 @@ def render_frontmatter(frontmatter: dict[str, Any]) -> str:
154
lines = ["---"]
155
for key, value in frontmatter.items():
156
lines.append(f"{key}: {yaml_value(value)}")
141
- lines.extend(["---", ""])
157
+ lines.extend(["---", "", ""])
158
return "\n".join(lines)
159
160
@@ -181,6 +197,18 @@ def repo_markdown(repo: str) -> str:
197
return f"[{repo}](https://github.com/{repo})"
198
199
200
+def extract_featured_repos(body: str, top_repo: str) -> tuple[str, ...]:
201
+ repos = [top_repo]
202
+ seen = {top_repo}
203
+ for match in REPO_LINK_PATTERN.finditer(body):
204
+ repo = match.group("repo")
205
+ if repo in seen:
206
+ continue
207
+ seen.add(repo)
208
+ repos.append(repo)
209
+ return tuple(repos)
210
+
211
+
212
def load_summary(path: Path) -> WeeklySummary:
213
frontmatter, body = analysis_gate.extract_frontmatter(path.read_text(encoding="utf-8"))
214
week = str(frontmatter.get("week", ""))
@@ -195,6 +223,7 @@ def load_summary(path: Path) -> WeeklySummary:
223
noise = get_subsection_text(trend_analysis, "Noise")
224
gaps = get_subsection_text(get_section_text(body, "What's Missing"), "Gaps")
225
conclusion = normalize_text(get_section_text(body, "Conclusion"))
226
+ top_repo = str(frontmatter["top_repo"])
227
228
return WeeklySummary(
229
source_path=path,
@@ -205,7 +234,8 @@ def load_summary(path: Path) -> WeeklySummary:
234
month=month,
235
tags=tuple(str(tag) for tag in frontmatter.get("tags", [])),
236
repos_featured=int(frontmatter["repos_featured"]),
208
- top_repo=str(frontmatter["top_repo"]),
237
+ top_repo=top_repo,
238
+ featured_repos=extract_featured_repos(body, top_repo),
239
summary=normalize_text(str(frontmatter["summary"])),
240
signal=signal,
241
noise=noise,
@@ -216,8 +246,6 @@ def load_summary(path: Path) -> WeeklySummary:
246
247
def load_weekly_summaries(analyzed_dir: Path) -> list[WeeklySummary]:
248
summaries = [load_summary(path) for path in sorted(analyzed_dir.glob(f"*{SUMMARY_SUFFIX}"))]
219
- if not summaries:
220
- raise RollupError(f"No weekly summaries found in {analyzed_dir}")
249
return sorted(summaries, key=lambda item: (item.date, item.week))
250
251
@@ -320,9 +348,10 @@ def build_monthly_pages(summaries: list[WeeklySummary], content_root: Path) -> l
348
pages: list[RollupPage] = []
349
for (year, month), items in sorted(grouped.items()):
350
items = sorted(items, key=lambda item: (item.date, item.week))
323
- tags_counter: Counter[str] = Counter(tag for item in items for tag in item.tags)
351
+ tags_counter: Counter[str] = Counter()
352
page_entries: dict[str, list[RollupEntry]] = {section: [] for section in MONTHLY_SECTIONS}
353
for item in items:
354
+ tags_counter.update(item.tags)
355
for section, entry in monthly_entries(item, tags_counter).items():
356
page_entries[section].append(entry)
357
pages.append(
@@ -335,7 +364,7 @@ def build_monthly_pages(summaries: list[WeeklySummary], content_root: Path) -> l
364
"year": year,
365
"categories": ["monthly"],
366
"weeks_covered": [item.week for item in items],
338
- "total_repos_featured": sum(item.repos_featured for item in items),
367
+ "total_repos_featured": len({repo for item in items for repo in item.featured_repos}),
368
},
369
sections=page_entries,
370
section_order=MONTHLY_SECTIONS,
@@ -352,9 +381,10 @@ def build_yearly_pages(summaries: list[WeeklySummary], content_root: Path) -> li
381
pages: list[RollupPage] = []
382
for year, items in sorted(grouped.items()):
383
items = sorted(items, key=lambda item: (item.date, item.week))
355
- tags_counter: Counter[str] = Counter(tag for item in items for tag in item.tags)
384
+ tags_counter: Counter[str] = Counter()
385
page_entries: dict[str, list[RollupEntry]] = {section: [] for section in YEARLY_SECTIONS}
386
for item in items:
387
+ tags_counter.update(item.tags)
388
for section, entry in yearly_entries(item, tags_counter).items():
389
page_entries[section].append(entry)
390
months_covered = sorted({item.month_slug for item in items})
@@ -389,13 +419,19 @@ def merge_sections(path: Path, section_order: list[str], new_entries: dict[str,
419
rendered_sections: list[str] = []
420
for section in section_order:
421
content = existing_sections.get(section, "")
422
+ if content.strip() == NO_UPDATES_PLACEHOLDER:
423
+ content = ""
424
for entry in new_entries[section]:
425
if entry.marker in content:
426
continue
427
content = f"{content.rstrip()}\n\n{entry.text}" if content.strip() else entry.text
396
- section_body = content.strip()
397
- if not section_body:
398
- section_body = "_No updates yet._"
428
+ section_body = content.strip() or NO_UPDATES_PLACEHOLDER
429
+ rendered_sections.append(f"## {section}\n\n{section_body}")
430
+
431
+ for section, content in existing_sections.items():
432
+ if section in section_order:
433
+ continue
434
+ section_body = content.strip() or NO_UPDATES_PLACEHOLDER
435
rendered_sections.append(f"## {section}\n\n{section_body}")
436
437
if intro.strip():
@@ -411,6 +447,9 @@ def write_rollup(page: RollupPage) -> None:
447
448
def generate_rollups(analyzed_dir: Path, content_root: Path) -> list[Path]:
449
summaries = load_weekly_summaries(analyzed_dir)
450
+ if not summaries:
451
+ return []
452
+
453
written: list[Path] = []
454
for page in [*build_monthly_pages(summaries, content_root), *build_yearly_pages(summaries, content_root)]:
455
write_rollup(page)
@@ -421,6 +460,9 @@ def generate_rollups(analyzed_dir: Path, content_root: Path) -> list[Path]:
460
def main(argv: list[str] | None = None) -> int:
461
args = parse_args(argv)
462
written = generate_rollups(args.analyzed_dir, args.content_root)
463
+ if not written:
464
+ print(f"No weekly summaries found in {args.analyzed_dir}; skipping rollup generation.", file=sys.stderr)
465
+ return 0
466
for path in written:
467
print(f"Generated {path}")
468
return 0
tests/test_generate_rollups.py
+96
-9
@@ -1,3 +1,4 @@
1
+import io
2
import tempfile
3
import unittest
4
from pathlib import Path
@@ -5,14 +6,34 @@ from pathlib import Path
6
import scripts.generate_rollups as generate_rollups
7
8
8
-def make_summary(*, week: str, date: str, top_repo: str, summary: str, signal: str, noise: str, gaps: str, conclusion: str) -> str:
9
+WORKSPACE_ROOT = Path(".test-workspaces")
10
+
11
+
12
+def make_summary(
13
+ *,
14
+ week: str,
15
+ date: str,
16
+ top_repo: str,
17
+ summary: str,
18
+ signal: str,
19
+ noise: str,
20
+ gaps: str,
21
+ conclusion: str,
22
+ tags: tuple[str, ...] = ("ai", "agents", "developer-tooling"),
23
+ repo_mentions: tuple[str, ...] = (),
24
+) -> str:
25
year = int(week[:4])
26
+ rendered_tags = ", ".join(tags)
27
+ linked_mentions = " ".join(
28
+ f"[{repo}](https://github.com/{repo}) is part of the weekly conversation." for repo in repo_mentions
29
+ )
30
+ notable_new = f"A fresh set of launches landed. {linked_mentions}".strip()
31
return f'''---
32
title: "Week {int(week[-2:])}, {year} Analysis"
33
date: {date}
34
week: "{week}"
35
year: {year}
15
-tags: [ai, agents, developer-tooling]
36
+tags: [{rendered_tags}]
37
categories: [weekly]
38
repos_featured: 10
39
top_repo: "{top_repo}"
@@ -23,7 +44,7 @@ stars_tracked: 1000
44
45
## Notable New Repositories
46
26
-A fresh set of launches landed.
47
+{notable_new}
48
49
## Trending This Week
50
@@ -51,10 +72,14 @@ Momentum concentrated around practical tooling.
72
'''
73
74
75
+def temporary_workspace() -> tempfile.TemporaryDirectory[str]:
76
+ WORKSPACE_ROOT.mkdir(exist_ok=True)
77
+ return tempfile.TemporaryDirectory(dir=WORKSPACE_ROOT.resolve())
78
+
79
+
80
class GenerateRollupsTests(unittest.TestCase):
81
def test_generate_rollups_creates_monthly_and_yearly_pages(self) -> None:
56
- tests_root = Path(__file__).resolve().parent
57
- with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
82
+ with temporary_workspace() as tmpdir:
83
base = Path(tmpdir)
84
analyzed_dir = base / "data" / "analyzed"
85
content_root = base / "content"
@@ -84,7 +109,8 @@ class GenerateRollupsTests(unittest.TestCase):
109
self.assertIn('title: "May 2026 Rollup"', monthly)
110
self.assertIn('categories: ["monthly"]', monthly)
111
self.assertIn('weeks_covered: ["2026-W21"]', monthly)
87
- self.assertIn('total_repos_featured: 10', monthly)
112
+ self.assertIn('total_repos_featured: 1', monthly)
113
+ self.assertIn('---\n\n## Month Overview', monthly)
114
self.assertIn('## Month Overview', monthly)
115
self.assertIn('### Week 2026-W21', monthly)
116
self.assertIn('[Week 21, 2026](/weekly/2026/W21/)', monthly)
@@ -99,8 +125,7 @@ class GenerateRollupsTests(unittest.TestCase):
125
self.assertIn('[May 2026](/monthly/2026/05/)', yearly)
126
127
def test_generate_rollups_is_append_only_for_existing_pages(self) -> None:
102
- tests_root = Path(__file__).resolve().parent
103
- with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir:
128
+ with temporary_workspace() as tmpdir:
129
base = Path(tmpdir)
130
analyzed_dir = base / "data" / "analyzed"
131
content_root = base / "content"
@@ -116,6 +141,8 @@ class GenerateRollupsTests(unittest.TestCase):
141
noise="Exploit-heavy projects still added editorial noise.",
142
gaps="Reliable momentum data remained missing.",
143
conclusion="The strongest projects made automation safer to adopt.",
144
+ tags=("alpha",),
145
+ repo_mentions=("octo/shared-kit",),
146
),
147
encoding="utf-8",
148
)
@@ -135,6 +162,8 @@ class GenerateRollupsTests(unittest.TestCase):
162
noise="Wrapper projects still outnumbered differentiated platforms.",
163
gaps="Defensive tooling still lagged behind orchestration tools.",
164
conclusion="The durable winners reduced toil without hiding trade-offs.",
165
+ tags=("beta",),
166
+ repo_mentions=("octo/shared-kit", "octo/deploy-guard"),
167
),
168
encoding="utf-8",
169
)
@@ -144,14 +173,19 @@ class GenerateRollupsTests(unittest.TestCase):
173
second_yearly = yearly_path.read_text(encoding="utf-8")
174
175
self.assertIn('weeks_covered: ["2026-W21", "2026-W22"]', second_monthly)
176
+ self.assertIn('total_repos_featured: 4', second_monthly)
177
self.assertIn('months_covered: ["2026-05"]', second_yearly)
178
for expected in [
179
'### Week 2026-W21 — [Week 21, 2026](/weekly/2026/W21/)',
180
'- [octo/signal-kit](https://github.com/octo/signal-kit) led the published weekly analysis for 2026-W21.',
181
'- Signal: Teams preferred operational automation over generic hype.',
182
'- Gap to watch: Reliable momentum data remained missing.',
183
+ '- Recurring themes so far: alpha.',
184
+ '- Themes in rotation: alpha.',
185
]:
154
- self.assertIn(expected, second_monthly)
186
+ self.assertIn(expected, second_monthly if 'Recurring themes' in expected else second_yearly if 'Themes in rotation' in expected else second_monthly)
187
+ self.assertIn('- Recurring themes so far: alpha, beta.', second_monthly)
188
+ self.assertIn('- Themes in rotation: alpha, beta.', second_yearly)
189
for expected in [
190
'### May 2026 update — 2026-W21',
191
'- [May 2026](/monthly/2026/05/) gained a new weekly signal via [Week 21, 2026](/weekly/2026/W21/).',
@@ -166,6 +200,59 @@ class GenerateRollupsTests(unittest.TestCase):
200
self.assertNotEqual(first_monthly, second_monthly)
201
self.assertNotEqual(first_yearly, second_yearly)
202
203
+ def test_generate_rollups_replaces_placeholder_and_preserves_unknown_sections(self) -> None:
204
+ with temporary_workspace() as tmpdir:
205
+ base = Path(tmpdir)
206
+ analyzed_dir = base / "data" / "analyzed"
207
+ content_root = base / "content"
208
+ analyzed_dir.mkdir(parents=True)
209
+ monthly_path = content_root / "monthly" / "2026" / "05.md"
210
+ monthly_path.parent.mkdir(parents=True, exist_ok=True)
211
+ monthly_path.write_text(
212
+ "---\ntitle: \"May 2026 Rollup\"\n---\n\n## Month Overview\n\n_No updates yet._\n\n## Legacy Notes\n\nKeep this section.\n",
213
+ encoding="utf-8",
214
+ )
215
+
216
+ (analyzed_dir / "2026-W21-summary.md").write_text(
217
+ make_summary(
218
+ week="2026-W21",
219
+ date="2026-05-18T12:07:20+00:00",
220
+ top_repo="octo/signal-kit",
221
+ summary="Practical agent tooling led the week.",
222
+ signal="Teams preferred operational automation over generic hype.",
223
+ noise="Exploit-heavy projects still added editorial noise.",
224
+ gaps="Reliable momentum data remained missing.",
225
+ conclusion="The strongest projects made automation safer to adopt.",
226
+ ),
227
+ encoding="utf-8",
228
+ )
229
+
230
+ generate_rollups.generate_rollups(analyzed_dir, content_root)
231
+ monthly = monthly_path.read_text(encoding="utf-8")
232
+
233
+ self.assertNotIn("_No updates yet._\n\n### Week 2026-W21", monthly)
234
+ self.assertIn("### Week 2026-W21", monthly)
235
+ self.assertIn("## Legacy Notes\n\nKeep this section.", monthly)
236
+
237
+ def test_generate_rollups_returns_empty_when_no_summaries_exist(self) -> None:
238
+ with temporary_workspace() as tmpdir:
239
+ base = Path(tmpdir)
240
+ analyzed_dir = base / "data" / "analyzed"
241
+ content_root = base / "content"
242
+ analyzed_dir.mkdir(parents=True)
243
+
244
+ self.assertEqual(generate_rollups.generate_rollups(analyzed_dir, content_root), [])
245
+ stderr = io.StringIO()
246
+ with unittest.mock.patch("sys.stderr", stderr):
247
+ self.assertEqual(generate_rollups.main(["--analyzed-dir", str(analyzed_dir), "--content-root", str(content_root)]), 0)
248
+ self.assertIn("No weekly summaries found", stderr.getvalue())
249
+
250
+ def test_parse_args_defaults_resolve_from_project_root(self) -> None:
251
+ args = generate_rollups.parse_args([])
252
+
253
+ self.assertEqual(args.analyzed_dir, Path(generate_rollups.PROJECT_ROOT / "data" / "analyzed"))
254
+ self.assertEqual(args.content_root, Path(generate_rollups.PROJECT_ROOT / "content"))
255
+
256
257
if __name__ == "__main__":
258
unittest.main()
tests/test_pipeline.py
+6
-2
@@ -203,8 +203,12 @@ class WorkflowConfigTests(unittest.TestCase):
203
commit_step = next((s for s in generate_job["steps"] if s.get("name") == "Commit generated content"), None)
204
self.assertIsNotNone(commit_step)
205
commit_run = commit_step["run"]
206
- self.assertIn("content/weekly content/monthly content/yearly", commit_run)
207
- self.assertIn("git add content/weekly/ content/monthly/ content/yearly/", commit_run)
206
+ self.assertIn("content/weekly", commit_run)
207
+ self.assertIn("content/monthly", commit_run)
208
+ self.assertIn("content/yearly", commit_run)
209
+ self.assertIn("git add content/weekly/", commit_run)
210
+ self.assertIn("content/monthly/", commit_run)
211
+ self.assertIn("content/yearly/", commit_run)
212
213
upload_step = next((s for s in generate_job["steps"] if s.get("name") == "Upload generated content artifact"), None)
214
self.assertIsNotNone(upload_step)