1
+#!/usr/bin/env python3
2
+from __future__ import annotations
3
+
4
+import argparse
5
+import re
6
+import sys
7
+from dataclasses import dataclass
8
+from pathlib import Path
9
+from typing import Any, Iterable
10
+
11
+if __package__ in {None, ""}:
12
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
13
+
14
+import scripts.analysis_gate as analysis_gate
15
+
16
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
17
+DEFAULT_CONTENT_ROOT = PROJECT_ROOT / "content"
18
+MONTH_SECTION_PATTERN = re.compile(r"(?m)^##\s+(.+?)\s*$")
19
+WEEK_BLOCK_PATTERN = re.compile(r"(?ms)^###\s+.+?\s*$\n(.*?)(?=^###\s+|\Z)")
20
+LINK_PATTERN = re.compile(r"\[([^\]]+)\]\([^)]+\)")
21
+WORD_PATTERN = re.compile(r"\S+")
22
+
23
+MONTH_NAMES = {
24
+ 1: "January",
25
+ 2: "February",
26
+ 3: "March",
27
+ 4: "April",
28
+ 5: "May",
29
+ 6: "June",
30
+ 7: "July",
31
+ 8: "August",
32
+ 9: "September",
33
+ 10: "October",
34
+ 11: "November",
35
+ 12: "December",
36
+}
37
+
38
+STOPWORDS = {
39
+ "a",
40
+ "an",
41
+ "and",
42
+ "are",
43
+ "as",
44
+ "at",
45
+ "be",
46
+ "by",
47
+ "for",
48
+ "from",
49
+ "has",
50
+ "in",
51
+ "into",
52
+ "is",
53
+ "it",
54
+ "its",
55
+ "of",
56
+ "on",
57
+ "or",
58
+ "that",
59
+ "the",
60
+ "their",
61
+ "this",
62
+ "to",
63
+ "was",
64
+ "were",
65
+ "while",
66
+ "with",
67
+}
68
+
69
+
70
+@dataclass(frozen=True)
71
+class MonthSnapshot:
72
+ path: Path
73
+ year: int
74
+ month: int
75
+ title: str
76
+ date: str
77
+ summaries: tuple[str, ...]
78
+ themes: tuple[str, ...]
79
+ signals: tuple[str, ...]
80
+ noise: tuple[str, ...]
81
+ gaps: tuple[str, ...]
82
+ closing_reads: tuple[str, ...]
83
+
84
+ @property
85
+ def month_name(self) -> str:
86
+ return MONTH_NAMES[self.month]
87
+
88
+ @property
89
+ def month_slug(self) -> str:
90
+ return f"{self.year}-{self.month:02d}"
91
+
92
+ @property
93
+ def link(self) -> str:
94
+ return f"/monthly/{self.year}/{self.month:02d}/"
95
+
96
+ @property
97
+ def text_blob(self) -> str:
98
+ return " ".join(
99
+ [
100
+ self.title,
101
+ *self.summaries,
102
+ *self.themes,
103
+ *self.signals,
104
+ *self.noise,
105
+ *self.gaps,
106
+ *self.closing_reads,
107
+ ]
108
+ )
109
+
110
+
111
+@dataclass(frozen=True)
112
+class YearlyNarrativePage:
113
+ year: int
114
+ path: Path
115
+ frontmatter: dict[str, Any]
116
+ narrative: str
117
+ arc_lines: tuple[str, ...]
118
+
119
+
120
+@dataclass(frozen=True)
121
+class TrendFamily:
122
+ key: str
123
+ label: str
124
+ keywords: tuple[str, ...]
125
+ stages: tuple[tuple[str, tuple[str, ...]], ...]
126
+
127
+
128
+TREND_FAMILIES = (
129
+ TrendFamily(
130
+ key="agent-skills",
131
+ label="agent-skills",
132
+ keywords=("agent skill", "agent-skills", "skills pack", "skill package", "skill"),
133
+ stages=(
134
+ ("infrastructure", ("maturing", "infrastructure", "mcp", "small model", "small-model")),
135
+ ("economy", ("economy", "distribution format", "marketplace", "skills layer", "skills packs")),
136
+ (
137
+ "globalization",
138
+ ("east asian", "chinese", "global", "globalization", "xiaohongshu", "wechat", "cultural", "linguistic"),
139
+ ),
140
+ (
141
+ "verticalization",
142
+ ("verticalization", "vertical", "domain-specific", "role-specific", "legal", "medical", "finance", "education"),
143
+ ),
144
+ ),
145
+ ),
146
+ TrendFamily(
147
+ key="platform-gaming",
148
+ label="platform-gaming",
149
+ keywords=("star-farming", "fork inflation", "spam", "activator", "cheat", "prediction-market bot", "seo-farming"),
150
+ stages=(
151
+ ("star-farming", ("star-farming", "star farming", "seo-farming")),
152
+ ("fork-inflation", ("fork inflation", "fork-inflation", "inflated fork", "implausibly inflated")),
153
+ ("activator-spam", ("activator", "activated", "kms", "copy-trading", "keyword-repetition", "bot cluster")),
154
+ ("fraud-cheat noise", ("fraud", "wallet-spoofer", "game cheat", "crypto fraud", "software unlock", "prediction-market bot")),
155
+ ),
156
+ ),
157
+ TrendFamily(
158
+ key="security-gap",
159
+ label="security-gap",
160
+ keywords=("security gap", "prompt injection", "supply-chain", "supply chain", "agent execution security", "agent isolation", "permission-scoping"),
161
+ stages=(
162
+ ("identified", ("security signal", "security gap", "agent execution security", "permission-scoping", "agent isolation")),
163
+ ("widening", ("still holds", "remains", "widening", "become exploitable", "not attracting commensurate attention")),
164
+ ("unresolved", ("no tooling exists", "gap that will become exploitable", "does not exist", "stayed missing")),
165
+ ),
166
+ ),
167
+ TrendFamily(
168
+ key="self-hosted-ai",
169
+ label="self-hosted-ai",
170
+ keywords=("self-hosted", "local-sovereignty", "local sovereignty", "local-sovereignty", "billing friction", "workspace", "local-first"),
171
+ stages=(
172
+ ("friction", ("billing friction", "cost", "copilot billing")),
173
+ ("self-hosted workspaces", ("self-hosted", "workspace launch", "workspace")),
174
+ ("local sovereignty", ("local-sovereignty", "local sovereignty", "local-first", "sandboxd", "memory", "control")),
175
+ ),
176
+ ),
177
+)
178
+
179
+
180
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
181
+ parser = argparse.ArgumentParser(description="Generate yearly narrative pages from monthly rollups.")
182
+ parser.add_argument(
183
+ "--content-root",
184
+ type=Path,
185
+ default=DEFAULT_CONTENT_ROOT,
186
+ help="Root content directory containing monthly/ and yearly/.",
187
+ )
188
+ parser.add_argument(
189
+ "--year",
190
+ type=int,
191
+ action="append",
192
+ dest="years",
193
+ help="Optional year to regenerate. May be passed multiple times.",
194
+ )
195
+ return parser.parse_args(argv)
196
+
197
+
198
+def yaml_quote(value: str) -> str:
199
+ return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
200
+
201
+
202
+def yaml_value(value: Any) -> str:
203
+ if isinstance(value, str):
204
+ return yaml_quote(value)
205
+ if isinstance(value, bool):
206
+ return "true" if value else "false"
207
+ if isinstance(value, int):
208
+ return str(value)
209
+ if isinstance(value, list):
210
+ return f"[{', '.join(yaml_value(item) for item in value)}]"
211
+ return str(value)
212
+
213
+
214
+def render_frontmatter(frontmatter: dict[str, Any]) -> str:
215
+ lines = ["---"]
216
+ for key, value in frontmatter.items():
217
+ lines.append(f"{key}: {yaml_value(value)}")
218
+ lines.extend(["---", "", ""])
219
+ return "\n".join(lines)
220
+
221
+
222
+def strip_markdown(text: str) -> str:
223
+ cleaned = LINK_PATTERN.sub(r"\1", text)
224
+ cleaned = cleaned.replace("**", "").replace("*", "").replace("`", "")
225
+ return re.sub(r"\s+", " ", cleaned).strip()
226
+
227
+
228
+def split_sections(body: str) -> dict[str, str]:
229
+ matches = list(MONTH_SECTION_PATTERN.finditer(body))
230
+ sections: dict[str, str] = {}
231
+ for index, match in enumerate(matches):
232
+ start = match.end()
233
+ end = matches[index + 1].start() if index + 1 < len(matches) else len(body)
234
+ sections[match.group(1).strip()] = body[start:end].strip("\n")
235
+ return sections
236
+
237
+
238
+def extract_labeled_values(section_body: str, label: str) -> list[str]:
239
+ values: list[str] = []
240
+ for block in WEEK_BLOCK_PATTERN.finditer(section_body):
241
+ for line in block.group(1).splitlines():
242
+ if not line.startswith(f"- {label}:"):
243
+ continue
244
+ value = strip_markdown(line.split(":", 1)[1])
245
+ if value:
246
+ values.append(value)
247
+ return values
248
+
249
+
250
+def dedupe_preserving_order(values: Iterable[str]) -> list[str]:
251
+ result: list[str] = []
252
+ seen: set[str] = set()
253
+ for value in values:
254
+ normalized = value.strip()
255
+ if not normalized or normalized in seen:
256
+ continue
257
+ seen.add(normalized)
258
+ result.append(normalized)
259
+ return result
260
+
261
+
262
+def load_month_snapshot(path: Path) -> MonthSnapshot:
263
+ frontmatter, body = analysis_gate.extract_frontmatter(path.read_text(encoding="utf-8"))
264
+ sections = split_sections(body)
265
+ themes: list[str] = []
266
+ for raw in extract_labeled_values(sections.get("Month Overview", ""), "Recurring themes so far"):
267
+ themes.extend(part.strip() for part in raw.rstrip(".").split(",") if part.strip())
268
+ return MonthSnapshot(
269
+ path=path,
270
+ year=int(frontmatter["year"]),
271
+ month=int(frontmatter["month"]),
272
+ title=str(frontmatter.get("title", path.stem)),
273
+ date=str(frontmatter["date"]),
274
+ summaries=tuple(extract_labeled_values(sections.get("Month Overview", ""), "Summary")),
275
+ themes=tuple(dedupe_preserving_order(themes)),
276
+ signals=tuple(extract_labeled_values(sections.get("Trends Observed", ""), "Signal")),
277
+ noise=tuple(extract_labeled_values(sections.get("Trends Observed", ""), "Noise")),
278
+ gaps=tuple(extract_labeled_values(sections.get("Key Takeaways", ""), "Gap to watch")),
279
+ closing_reads=tuple(extract_labeled_values(sections.get("Key Takeaways", ""), "Closing read")),
280
+ )
281
+
282
+
283
+def load_month_snapshots(content_root: Path, years: Iterable[int] | None = None) -> list[MonthSnapshot]:
284
+ if years:
285
+ paths = []
286
+ for year in sorted(set(years)):
287
+ paths.extend(sorted((content_root / "monthly" / str(year)).glob("*.md")))
288
+ else:
289
+ paths = sorted((content_root / "monthly").glob("*/*.md"))
290
+ snapshots = [load_month_snapshot(path) for path in paths if path.is_file()]
291
+ return sorted(snapshots, key=lambda item: (item.year, item.month))
292
+
293
+
294
+def word_count(text: str) -> int:
295
+ return len(WORD_PATTERN.findall(text))
296
+
297
+
298
+def trim_words(text: str, limit: int) -> str:
299
+ words = text.split()
300
+ if len(words) <= limit:
301
+ return text.strip()
302
+ return " ".join(words[:limit]).rstrip(",;:.") + "…"
303
+
304
+
305
+def compress_phrase(text: str, limit: int = 24) -> str:
306
+ cleaned = strip_markdown(text)
307
+ cleaned = re.sub(r"^(Week \d+\s+|W\d+\s+)", "", cleaned)
308
+ cleaned = re.sub(r"^(The durable signal this week |This week |Week \d+ |W\d+ )", "", cleaned, flags=re.IGNORECASE)
309
+ cleaned = re.sub(r"\s+", " ", cleaned).strip().rstrip(".")
310
+ return trim_words(cleaned, limit)
311
+
312
+
313
+def keyword_score(text: str, keywords: Iterable[str]) -> int:
314
+ lowered = text.lower()
315
+ return sum(1 for keyword in keywords if keyword in lowered)
316
+
317
+
318
+def detect_family_arc(months: list[MonthSnapshot], family: TrendFamily) -> list[str]:
319
+ stages: list[str] = []
320
+ family_seen = False
321
+ for month in months:
322
+ lowered = month.text_blob.lower()
323
+ if keyword_score(lowered, family.keywords):
324
+ family_seen = True
325
+ for stage, keywords in family.stages:
326
+ if any(keyword in lowered for keyword in keywords) and stage not in stages:
327
+ stages.append(stage)
328
+ if family_seen and not stages:
329
+ stages.append("emerging")
330
+ return stages
331
+
332
+
333
+def build_theme_sentence(year: int, arcs: dict[str, list[str]]) -> str:
334
+ has_skills = bool(arcs.get("agent-skills"))
335
+ has_noise = bool(arcs.get("platform-gaming"))
336
+ has_security = bool(arcs.get("security-gap"))
337
+ has_local = bool(arcs.get("self-hosted-ai"))
338
+ if has_skills and has_noise:
339
+ sentence = (
340
+ f"{year} has been a split-screen story: agent tooling kept solidifying into a real distribution layer "
341
+ "while GitHub discovery got easier to game."
342
+ )
343
+ elif has_skills:
344
+ sentence = f"{year} has mainly been the year agent tooling stopped looking experimental and started behaving like infrastructure."
345
+ else:
346
+ sentence = f"{year} has so far been defined less by single launches than by shifts in how the ecosystem is organizing itself."
347
+ if has_security:
348
+ sentence += " The ecosystem moved faster on capability than on trust."
349
+ elif has_local:
350
+ sentence += " Control, cost, and local execution kept gaining weight."
351
+ return sentence
352
+
353
+
354
+def summarize_month(month: MonthSnapshot) -> str:
355
+ month_arcs = {family.key: detect_family_arc([month], family) for family in TREND_FAMILIES}
356
+ parts: list[str] = []
357
+
358
+ agent_arc = month_arcs.get("agent-skills", [])
359
+ if "globalization" in agent_arc and "verticalization" in agent_arc:
360
+ parts.append("agent skills globalized and started splitting into tighter verticals")
361
+ elif "economy" in agent_arc and "infrastructure" in agent_arc:
362
+ parts.append("agent skills hardened from plumbing into an economy")
363
+ elif "economy" in agent_arc:
364
+ parts.append("agent skills started looking like a real market layer")
365
+ elif "infrastructure" in agent_arc:
366
+ parts.append("agent tooling kept hardening into infrastructure")
367
+
368
+ local_arc = month_arcs.get("self-hosted-ai", [])
369
+ if "local sovereignty" in local_arc:
370
+ parts.append("self-hosted and local-sovereignty tools gained real momentum")
371
+ elif "self-hosted workspaces" in local_arc:
372
+ parts.append("self-hosted AI workspaces became more credible")
373
+
374
+ if month_arcs.get("security-gap"):
375
+ parts.append("the security gap stayed more visible than the fixes")
376
+
377
+ platform_arc = month_arcs.get("platform-gaming", [])
378
+ if "fork-inflation" in platform_arc:
379
+ parts.append("fork inflation replaced the earlier star-farming playbook")
380
+ elif "star-farming" in platform_arc:
381
+ parts.append("coordinated star-farming made discovery harder to trust")
382
+
383
+ if parts:
384
+ return "; ".join(parts[:-1]) + ("" if len(parts) < 2 else "; ") + parts[-1] if len(parts) > 1 else parts[0]
385
+
386
+ return trim_words(strip_markdown(month.summaries[-1] if month.summaries else month.text_blob), 32)
387
+
388
+
389
+def build_month_story(months: list[MonthSnapshot]) -> str:
390
+ sentences: list[str] = []
391
+ for month in months:
392
+ summary = summarize_month(month)
393
+ sentences.append(f"In {month.month_name}, {summary}")
394
+ if not sentences:
395
+ return ""
396
+ return " ".join(sentence.rstrip(".") + "." for sentence in sentences)
397
+
398
+
399
+def build_arc_commentary(arcs: dict[str, list[str]]) -> list[str]:
400
+ commentary: list[str] = []
401
+ agent_arc = arcs.get("agent-skills", [])
402
+ if agent_arc:
403
+ commentary.append(f"Agent skills moved through {' → '.join(agent_arc)}.")
404
+ platform_arc = arcs.get("platform-gaming", [])
405
+ if platform_arc:
406
+ commentary.append(f"Platform gaming adapted through {' → '.join(platform_arc)} instead of disappearing.")
407
+ local_arc = arcs.get("self-hosted-ai", [])
408
+ if local_arc:
409
+ commentary.append(f"Self-hosted AI evolved through {' → '.join(local_arc)} as builders chased more control over execution and cost.")
410
+ security_arc = arcs.get("security-gap", [])
411
+ if security_arc:
412
+ commentary.append("The security gap stayed ahead of the fixes: each month made the need for agent isolation, supply-chain auditing, and prompt-injection defenses easier to see.")
413
+ return commentary
414
+
415
+
416
+def build_prediction_review(arcs: dict[str, list[str]]) -> str:
417
+ confirmations: list[str] = []
418
+ if "globalization" in arcs.get("agent-skills", []):
419
+ confirmations.append("skills did globalize")
420
+ if "verticalization" in arcs.get("agent-skills", []):
421
+ confirmations.append("skills also verticalized quickly")
422
+ if len(arcs.get("platform-gaming", [])) >= 2:
423
+ confirmations.append("discovery-layer abuse mutated instead of self-correcting")
424
+ if arcs.get("self-hosted-ai"):
425
+ confirmations.append("local and self-hosted AI kept becoming a category rather than a workaround")
426
+ if arcs.get("security-gap"):
427
+ confirmations.append("the trust and security gap remained open")
428
+ if not confirmations:
429
+ return "The running predictions stayed directionally useful: the biggest structural questions still look unresolved."
430
+ joined = "; ".join(confirmations[:-1]) + ("" if len(confirmations) < 2 else "; ") + confirmations[-1] if len(confirmations) > 1 else confirmations[0]
431
+ return f"The running predictions were mostly right: {joined}."
432
+
433
+
434
+def compress_narrative(paragraphs: list[str], max_words: int = 500) -> str:
435
+ text = "\n\n".join(paragraph.strip() for paragraph in paragraphs if paragraph.strip())
436
+ if word_count(text) <= max_words:
437
+ return text
438
+ compressed = text
439
+ for limit in (460, 430, 400, 360):
440
+ words = compressed.split()
441
+ if len(words) <= max_words:
442
+ break
443
+ compressed = " ".join(words[:limit]).rstrip(",;:.") + "…"
444
+ return compressed
445
+
446
+
447
+def build_arc_lines(months: list[MonthSnapshot]) -> tuple[str, ...]:
448
+ arcs: list[str] = []
449
+ for family in TREND_FAMILIES:
450
+ stages = detect_family_arc(months, family)
451
+ if stages:
452
+ arcs.append(f"{family.label}: {' > '.join(stages)}")
453
+ return tuple(arcs)
454
+
455
+
456
+def synthesize_year(months: list[MonthSnapshot]) -> tuple[str, tuple[str, ...]]:
457
+ arcs = {family.key: detect_family_arc(months, family) for family in TREND_FAMILIES}
458
+ paragraphs = [
459
+ build_theme_sentence(months[0].year, arcs),
460
+ build_month_story(months),
461
+ " ".join(build_arc_commentary(arcs)),
462
+ build_prediction_review(arcs),
463
+ ]
464
+ return compress_narrative(paragraphs), build_arc_lines(months)
465
+
466
+
467
+def build_yearly_narrative_pages(content_root: Path, years: Iterable[int] | None = None) -> list[YearlyNarrativePage]:
468
+ grouped: dict[int, list[MonthSnapshot]] = {}
469
+ for snapshot in load_month_snapshots(content_root, years):
470
+ grouped.setdefault(snapshot.year, []).append(snapshot)
471
+
472
+ pages: list[YearlyNarrativePage] = []
473
+ for year, months in sorted(grouped.items()):
474
+ ordered = sorted(months, key=lambda item: item.month)
475
+ narrative, arc_lines = synthesize_year(ordered)
476
+ pages.append(
477
+ YearlyNarrativePage(
478
+ year=year,
479
+ path=content_root / "yearly" / f"{year}.md",
480
+ frontmatter={
481
+ "title": f"{year} Yearly Narrative",
482
+ "date": ordered[-1].date,
483
+ "year": year,
484
+ "categories": ["yearly"],
485
+ "months_covered": [month.month_slug for month in ordered],
486
+ "format": "narrative",
487
+ },
488
+ narrative=narrative,
489
+ arc_lines=arc_lines,
490
+ )
491
+ )
492
+ return pages
493
+
494
+
495
+def render_yearly_page(page: YearlyNarrativePage) -> str:
496
+ arc_body = "\n".join(f"- {line}" for line in page.arc_lines) if page.arc_lines else "_No updates yet._"
497
+ body = f"## Narrative\n\n{page.narrative}\n\n## Arc\n\n{arc_body}\n"
498
+ return render_frontmatter(page.frontmatter) + body
499
+
500
+
501
+def generate_yearly_narratives(content_root: Path, years: Iterable[int] | None = None) -> list[Path]:
502
+ written: list[Path] = []
503
+ for page in build_yearly_narrative_pages(content_root, years):
504
+ page.path.parent.mkdir(parents=True, exist_ok=True)
505
+ page.path.write_text(render_yearly_page(page), encoding="utf-8")
506
+ written.append(page.path)
507
+ return written
508
+
509
+
510
+def main(argv: list[str] | None = None) -> int:
511
+ args = parse_args(argv)
512
+ written = generate_yearly_narratives(args.content_root, args.years)
513
+ if not written:
514
+ print(f"No monthly rollups found under {args.content_root / 'monthly'}")
515
+ return 0
516
+ for path in written:
517
+ print(f"Generated {path}")
518
+ return 0
519
+
520
+
521
+if __name__ == "__main__":
522
+ raise SystemExit(main())