| 1 | #!/usr/bin/env python3 |
| 2 | """Generate yearly narrative pages from monthly rollup content. |
| 3 | |
| 4 | Reads monthly pages (content/monthly/YYYY/MM.md), detects trend-family arcs |
| 5 | across months, and synthesizes a cohesive year-in-review narrative. Output |
| 6 | includes: |
| 7 | - SEO-friendly editorial titles (max 70 chars) driven by detected arcs |
| 8 | - Meta description summaries (≤155 chars) extracted from narrative opening |
| 9 | - Cross-links to each contributing monthly report |
| 10 | - Structured frontmatter: months_covered, format, summary, categories |
| 11 | """ |
| 12 | |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import argparse |
| 16 | import re |
| 17 | import sys |
| 18 | from dataclasses import dataclass |
| 19 | from pathlib import Path |
| 20 | from typing import Any, Iterable |
| 21 | |
| 22 | if __package__ in {None, ""}: |
| 23 | sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) |
| 24 | |
| 25 | import scripts.analysis_gate as analysis_gate |
| 26 | |
| 27 | PROJECT_ROOT = Path(__file__).resolve().parent.parent |
| 28 | DEFAULT_CONTENT_ROOT = PROJECT_ROOT / "content" |
| 29 | MONTH_SECTION_PATTERN = re.compile(r"(?m)^##\s+(.+?)\s*$") |
| 30 | WEEK_BLOCK_PATTERN = re.compile(r"(?ms)^###\s+.+?\s*$\n(.*?)(?=^###\s+|\Z)") |
| 31 | LINK_PATTERN = re.compile(r"\[([^\]]+)\]\([^)]+\)") |
| 32 | WORD_PATTERN = re.compile(r"\S+") |
| 33 | HEADING_LINE_PATTERN = re.compile(r"(?m)^#+\s+") |
| 34 | |
| 35 | MONTH_NAMES = { |
| 36 | 1: "January", |
| 37 | 2: "February", |
| 38 | 3: "March", |
| 39 | 4: "April", |
| 40 | 5: "May", |
| 41 | 6: "June", |
| 42 | 7: "July", |
| 43 | 8: "August", |
| 44 | 9: "September", |
| 45 | 10: "October", |
| 46 | 11: "November", |
| 47 | 12: "December", |
| 48 | } |
| 49 | |
| 50 | STOPWORDS = { |
| 51 | "a", |
| 52 | "an", |
| 53 | "and", |
| 54 | "are", |
| 55 | "as", |
| 56 | "at", |
| 57 | "be", |
| 58 | "by", |
| 59 | "for", |
| 60 | "from", |
| 61 | "has", |
| 62 | "in", |
| 63 | "into", |
| 64 | "is", |
| 65 | "it", |
| 66 | "its", |
| 67 | "of", |
| 68 | "on", |
| 69 | "or", |
| 70 | "that", |
| 71 | "the", |
| 72 | "their", |
| 73 | "this", |
| 74 | "to", |
| 75 | "was", |
| 76 | "were", |
| 77 | "while", |
| 78 | "with", |
| 79 | } |
| 80 | |
| 81 | |
| 82 | @dataclass(frozen=True) |
| 83 | class MonthSnapshot: |
| 84 | path: Path |
| 85 | year: int |
| 86 | month: int |
| 87 | title: str |
| 88 | date: str |
| 89 | summaries: tuple[str, ...] |
| 90 | themes: tuple[str, ...] |
| 91 | signals: tuple[str, ...] |
| 92 | noise: tuple[str, ...] |
| 93 | gaps: tuple[str, ...] |
| 94 | closing_reads: tuple[str, ...] |
| 95 | synthesis_paragraphs: tuple[str, ...] = () |
| 96 | |
| 97 | @property |
| 98 | def month_name(self) -> str: |
| 99 | return MONTH_NAMES[self.month] |
| 100 | |
| 101 | @property |
| 102 | def month_slug(self) -> str: |
| 103 | return f"{self.year}-{self.month:02d}" |
| 104 | |
| 105 | @property |
| 106 | def link(self) -> str: |
| 107 | return f"/monthly/{self.year}/{self.month:02d}/" |
| 108 | |
| 109 | @property |
| 110 | def text_blob(self) -> str: |
| 111 | return " ".join( |
| 112 | [ |
| 113 | self.title, |
| 114 | *self.synthesis_paragraphs, |
| 115 | *self.summaries, |
| 116 | *self.themes, |
| 117 | *self.signals, |
| 118 | *self.noise, |
| 119 | *self.gaps, |
| 120 | *self.closing_reads, |
| 121 | ] |
| 122 | ) |
| 123 | |
| 124 | @property |
| 125 | def yearly_source_paragraphs(self) -> tuple[str, ...]: |
| 126 | if self.synthesis_paragraphs: |
| 127 | return self.synthesis_paragraphs |
| 128 | paragraphs = [ |
| 129 | sentence |
| 130 | for sentence in ( |
| 131 | self.summaries[-1] if self.summaries else "", |
| 132 | self.signals[-1] if self.signals else "", |
| 133 | self.gaps[-1] if self.gaps else "", |
| 134 | self.closing_reads[-1] if self.closing_reads else "", |
| 135 | ) |
| 136 | if sentence |
| 137 | ] |
| 138 | if paragraphs: |
| 139 | return tuple(paragraphs) |
| 140 | fallback = strip_markdown(self.text_blob) |
| 141 | return (fallback,) if fallback else () |
| 142 | |
| 143 | |
| 144 | @dataclass(frozen=True) |
| 145 | class YearlyNarrativePage: |
| 146 | year: int |
| 147 | path: Path |
| 148 | frontmatter: dict[str, Any] |
| 149 | narrative: str |
| 150 | |
| 151 | |
| 152 | @dataclass(frozen=True) |
| 153 | class TrendFamily: |
| 154 | key: str |
| 155 | label: str |
| 156 | keywords: tuple[str, ...] |
| 157 | stages: tuple[tuple[str, tuple[str, ...]], ...] |
| 158 | |
| 159 | |
| 160 | TREND_FAMILIES = ( |
| 161 | TrendFamily( |
| 162 | key="agent-skills", |
| 163 | label="agent-skills", |
| 164 | keywords=("agent skill", "agent-skills", "skills pack", "skill package", "skill"), |
| 165 | stages=( |
| 166 | ("infrastructure", ("maturing", "infrastructure", "mcp", "small model", "small-model")), |
| 167 | ( |
| 168 | "economy", |
| 169 | ("economy", "distribution format", "marketplace", "skills layer", "skills packs"), |
| 170 | ), |
| 171 | ( |
| 172 | "globalization", |
| 173 | ( |
| 174 | "east asian", |
| 175 | "chinese", |
| 176 | "global", |
| 177 | "globalization", |
| 178 | "xiaohongshu", |
| 179 | "wechat", |
| 180 | "cultural", |
| 181 | "linguistic", |
| 182 | ), |
| 183 | ), |
| 184 | ( |
| 185 | "verticalization", |
| 186 | ( |
| 187 | "verticalization", |
| 188 | "vertical", |
| 189 | "domain-specific", |
| 190 | "role-specific", |
| 191 | "legal", |
| 192 | "medical", |
| 193 | "finance", |
| 194 | "education", |
| 195 | ), |
| 196 | ), |
| 197 | ), |
| 198 | ), |
| 199 | TrendFamily( |
| 200 | key="platform-gaming", |
| 201 | label="platform-gaming", |
| 202 | keywords=( |
| 203 | "star-farming", |
| 204 | "fork inflation", |
| 205 | "spam", |
| 206 | "activator", |
| 207 | "cheat", |
| 208 | "prediction-market bot", |
| 209 | "seo-farming", |
| 210 | ), |
| 211 | stages=( |
| 212 | ("star-farming", ("star-farming", "star farming", "seo-farming")), |
| 213 | ( |
| 214 | "fork-inflation", |
| 215 | ("fork inflation", "fork-inflation", "inflated fork", "implausibly inflated"), |
| 216 | ), |
| 217 | ( |
| 218 | "activator-spam", |
| 219 | ( |
| 220 | "activator", |
| 221 | "activated", |
| 222 | "kms", |
| 223 | "copy-trading", |
| 224 | "keyword-repetition", |
| 225 | "bot cluster", |
| 226 | ), |
| 227 | ), |
| 228 | ( |
| 229 | "fraud-cheat noise", |
| 230 | ( |
| 231 | "fraud", |
| 232 | "wallet-spoofer", |
| 233 | "game cheat", |
| 234 | "crypto fraud", |
| 235 | "software unlock", |
| 236 | "prediction-market bot", |
| 237 | ), |
| 238 | ), |
| 239 | ), |
| 240 | ), |
| 241 | TrendFamily( |
| 242 | key="security-gap", |
| 243 | label="security-gap", |
| 244 | keywords=( |
| 245 | "security gap", |
| 246 | "prompt injection", |
| 247 | "supply-chain", |
| 248 | "supply chain", |
| 249 | "agent execution security", |
| 250 | "agent isolation", |
| 251 | "permission-scoping", |
| 252 | ), |
| 253 | stages=( |
| 254 | ( |
| 255 | "identified", |
| 256 | ( |
| 257 | "security signal", |
| 258 | "security gap", |
| 259 | "agent execution security", |
| 260 | "permission-scoping", |
| 261 | "agent isolation", |
| 262 | ), |
| 263 | ), |
| 264 | ( |
| 265 | "widening", |
| 266 | ( |
| 267 | "still holds", |
| 268 | "remains", |
| 269 | "widening", |
| 270 | "become exploitable", |
| 271 | "not attracting commensurate attention", |
| 272 | ), |
| 273 | ), |
| 274 | ( |
| 275 | "unresolved", |
| 276 | ( |
| 277 | "no tooling exists", |
| 278 | "gap that will become exploitable", |
| 279 | "does not exist", |
| 280 | "stayed missing", |
| 281 | ), |
| 282 | ), |
| 283 | ), |
| 284 | ), |
| 285 | TrendFamily( |
| 286 | key="self-hosted-ai", |
| 287 | label="self-hosted-ai", |
| 288 | keywords=( |
| 289 | "self-hosted", |
| 290 | "local-sovereignty", |
| 291 | "local sovereignty", |
| 292 | "local-sovereignty", |
| 293 | "billing friction", |
| 294 | "workspace", |
| 295 | "local-first", |
| 296 | ), |
| 297 | stages=( |
| 298 | ("friction", ("billing friction", "cost", "copilot billing")), |
| 299 | ("self-hosted workspaces", ("self-hosted", "workspace launch", "workspace")), |
| 300 | ( |
| 301 | "local sovereignty", |
| 302 | ( |
| 303 | "local-sovereignty", |
| 304 | "local sovereignty", |
| 305 | "local-first", |
| 306 | "sandboxd", |
| 307 | "memory", |
| 308 | "control", |
| 309 | ), |
| 310 | ), |
| 311 | ), |
| 312 | ), |
| 313 | ) |
| 314 | |
| 315 | |
| 316 | def parse_args(argv: list[str] | None = None) -> argparse.Namespace: |
| 317 | parser = argparse.ArgumentParser( |
| 318 | description="Generate yearly narrative pages from monthly rollups." |
| 319 | ) |
| 320 | parser.add_argument( |
| 321 | "--content-root", |
| 322 | type=Path, |
| 323 | default=DEFAULT_CONTENT_ROOT, |
| 324 | help="Root content directory containing monthly/ and yearly/.", |
| 325 | ) |
| 326 | parser.add_argument( |
| 327 | "--year", |
| 328 | type=int, |
| 329 | action="append", |
| 330 | dest="years", |
| 331 | help="Optional year to regenerate. May be passed multiple times.", |
| 332 | ) |
| 333 | return parser.parse_args(argv) |
| 334 | |
| 335 | |
| 336 | def yaml_quote(value: str) -> str: |
| 337 | return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' |
| 338 | |
| 339 | |
| 340 | def yaml_value(value: Any) -> str: |
| 341 | if isinstance(value, str): |
| 342 | return yaml_quote(value) |
| 343 | if isinstance(value, bool): |
| 344 | return "true" if value else "false" |
| 345 | if isinstance(value, int): |
| 346 | return str(value) |
| 347 | if isinstance(value, list): |
| 348 | return f"[{', '.join(yaml_value(item) for item in value)}]" |
| 349 | return str(value) |
| 350 | |
| 351 | |
| 352 | def render_frontmatter(frontmatter: dict[str, Any]) -> str: |
| 353 | lines = ["---"] |
| 354 | for key, value in frontmatter.items(): |
| 355 | lines.append(f"{key}: {yaml_value(value)}") |
| 356 | lines.extend(["---", "", ""]) |
| 357 | return "\n".join(lines) |
| 358 | |
| 359 | |
| 360 | def strip_markdown(text: str) -> str: |
| 361 | cleaned = LINK_PATTERN.sub(r"\1", text) |
| 362 | cleaned = cleaned.replace("**", "").replace("*", "").replace("`", "") |
| 363 | return re.sub(r"\s+", " ", cleaned).strip() |
| 364 | |
| 365 | |
| 366 | def split_sections(body: str) -> dict[str, str]: |
| 367 | matches = list(MONTH_SECTION_PATTERN.finditer(body)) |
| 368 | sections: dict[str, str] = {} |
| 369 | for index, match in enumerate(matches): |
| 370 | start = match.end() |
| 371 | end = matches[index + 1].start() if index + 1 < len(matches) else len(body) |
| 372 | sections[match.group(1).strip()] = body[start:end].strip("\n") |
| 373 | return sections |
| 374 | |
| 375 | |
| 376 | def extract_prose_paragraphs(body: str) -> tuple[str, ...]: |
| 377 | paragraphs: list[str] = [] |
| 378 | for block in re.split(r"\n\s*\n", body.strip()): |
| 379 | lines = [line.rstrip() for line in block.splitlines() if line.strip()] |
| 380 | if not lines: |
| 381 | continue |
| 382 | if all(line.lstrip().startswith("#") for line in lines): |
| 383 | continue |
| 384 | cleaned_lines: list[str] = [] |
| 385 | for line in lines: |
| 386 | stripped = line.strip() |
| 387 | if stripped.startswith("#"): |
| 388 | stripped = HEADING_LINE_PATTERN.sub("", stripped) |
| 389 | stripped = re.sub(r"^[-*]\s+", "", stripped) |
| 390 | stripped = re.sub(r"^\d+\.\s+", "", stripped) |
| 391 | if stripped: |
| 392 | cleaned_lines.append(stripped) |
| 393 | paragraph = strip_markdown(" ".join(cleaned_lines)) |
| 394 | if paragraph and paragraph != "_No updates yet._": |
| 395 | paragraphs.append(paragraph) |
| 396 | return tuple(paragraphs) |
| 397 | |
| 398 | |
| 399 | def extract_labeled_values(section_body: str, label: str) -> list[str]: |
| 400 | values: list[str] = [] |
| 401 | for block in WEEK_BLOCK_PATTERN.finditer(section_body): |
| 402 | for line in block.group(1).splitlines(): |
| 403 | if not line.startswith(f"- {label}:"): |
| 404 | continue |
| 405 | value = strip_markdown(line.split(":", 1)[1]) |
| 406 | if value: |
| 407 | values.append(value) |
| 408 | return values |
| 409 | |
| 410 | |
| 411 | def dedupe_preserving_order(values: Iterable[str]) -> list[str]: |
| 412 | result: list[str] = [] |
| 413 | seen: set[str] = set() |
| 414 | for value in values: |
| 415 | normalized = value.strip() |
| 416 | if not normalized or normalized in seen: |
| 417 | continue |
| 418 | seen.add(normalized) |
| 419 | result.append(normalized) |
| 420 | return result |
| 421 | |
| 422 | |
| 423 | def frontmatter_list(frontmatter: dict[str, Any], key: str) -> list[str]: |
| 424 | raw = frontmatter.get(key, []) |
| 425 | if isinstance(raw, list): |
| 426 | return [str(item) for item in raw if str(item).strip()] |
| 427 | return [] |
| 428 | |
| 429 | |
| 430 | def load_month_snapshot(path: Path) -> MonthSnapshot: |
| 431 | frontmatter, body = analysis_gate.extract_frontmatter(path.read_text(encoding="utf-8")) |
| 432 | sections = split_sections(body) |
| 433 | month_synthesis = strip_markdown(sections.get("Month Synthesis", "")) |
| 434 | if month_synthesis: |
| 435 | summaries = dedupe_preserving_order( |
| 436 | [str(frontmatter.get("summary", "")).strip(), month_synthesis] |
| 437 | ) |
| 438 | themes = dedupe_preserving_order( |
| 439 | frontmatter_list(frontmatter, "themes") |
| 440 | + frontmatter_list(frontmatter, "persistent_themes") |
| 441 | + frontmatter_list(frontmatter, "accelerating_themes") |
| 442 | + frontmatter_list(frontmatter, "weakening_themes") |
| 443 | ) |
| 444 | signals = dedupe_preserving_order( |
| 445 | [strip_markdown(sections.get("Trend Arc", ""))] |
| 446 | + [ |
| 447 | theme.replace("-", " ") |
| 448 | for theme in frontmatter_list(frontmatter, "accelerating_themes") |
| 449 | ] |
| 450 | + [ |
| 451 | theme.replace("-", " ") |
| 452 | for theme in frontmatter_list(frontmatter, "persistent_themes") |
| 453 | ] |
| 454 | ) |
| 455 | noise = tuple( |
| 456 | theme.replace("-", " ") for theme in frontmatter_list(frontmatter, "weakening_themes") |
| 457 | ) |
| 458 | gaps = tuple(frontmatter_list(frontmatter, "key_gaps")) |
| 459 | closing_reads = tuple( |
| 460 | value |
| 461 | for value in [ |
| 462 | strip_markdown(sections.get("Prediction Review", "")), |
| 463 | str(frontmatter.get("summary", "")).strip(), |
| 464 | ] |
| 465 | if value |
| 466 | ) |
| 467 | return MonthSnapshot( |
| 468 | path=path, |
| 469 | year=int(frontmatter["year"]), |
| 470 | month=int(frontmatter["month"]), |
| 471 | title=str(frontmatter.get("title", path.stem)), |
| 472 | date=str(frontmatter["date"]), |
| 473 | summaries=tuple(summaries), |
| 474 | themes=tuple(themes), |
| 475 | signals=tuple(signals), |
| 476 | noise=noise, |
| 477 | gaps=gaps, |
| 478 | closing_reads=closing_reads, |
| 479 | ) |
| 480 | |
| 481 | themes: list[str] = [] |
| 482 | for raw in extract_labeled_values( |
| 483 | sections.get("Month Overview", ""), "Recurring themes so far" |
| 484 | ): |
| 485 | themes.extend(part.strip() for part in raw.rstrip(".").split(",") if part.strip()) |
| 486 | return MonthSnapshot( |
| 487 | path=path, |
| 488 | year=int(frontmatter["year"]), |
| 489 | month=int(frontmatter["month"]), |
| 490 | title=str(frontmatter.get("title", path.stem)), |
| 491 | date=str(frontmatter["date"]), |
| 492 | summaries=tuple(extract_labeled_values(sections.get("Month Overview", ""), "Summary")), |
| 493 | themes=tuple(dedupe_preserving_order(themes)), |
| 494 | signals=tuple(extract_labeled_values(sections.get("Trends Observed", ""), "Signal")), |
| 495 | noise=tuple(extract_labeled_values(sections.get("Trends Observed", ""), "Noise")), |
| 496 | gaps=tuple(extract_labeled_values(sections.get("Key Takeaways", ""), "Gap to watch")), |
| 497 | closing_reads=tuple( |
| 498 | extract_labeled_values(sections.get("Key Takeaways", ""), "Closing read") |
| 499 | ), |
| 500 | ) |
| 501 | |
| 502 | |
| 503 | def analyzed_dir_for(content_root: Path) -> Path: |
| 504 | return content_root.parent / "data" / "analyzed" |
| 505 | |
| 506 | |
| 507 | def load_month_synthesis_paragraphs(path: Path) -> tuple[str, ...]: |
| 508 | text = path.read_text(encoding="utf-8") |
| 509 | try: |
| 510 | _, body = analysis_gate.extract_frontmatter(text) |
| 511 | except ValueError: |
| 512 | body = text |
| 513 | return extract_prose_paragraphs(body) |
| 514 | |
| 515 | |
| 516 | def load_month_snapshot_with_preference(path: Path, content_root: Path) -> MonthSnapshot: |
| 517 | snapshot = load_month_snapshot(path) |
| 518 | synthesis_path = ( |
| 519 | analyzed_dir_for(content_root) / f"{snapshot.year}-{snapshot.month:02d}-month-synthesis.md" |
| 520 | ) |
| 521 | if not synthesis_path.is_file(): |
| 522 | return snapshot |
| 523 | synthesis_paragraphs = load_month_synthesis_paragraphs(synthesis_path) |
| 524 | if not synthesis_paragraphs: |
| 525 | return snapshot |
| 526 | return MonthSnapshot( |
| 527 | path=snapshot.path, |
| 528 | year=snapshot.year, |
| 529 | month=snapshot.month, |
| 530 | title=snapshot.title, |
| 531 | date=snapshot.date, |
| 532 | summaries=snapshot.summaries, |
| 533 | themes=snapshot.themes, |
| 534 | signals=snapshot.signals, |
| 535 | noise=snapshot.noise, |
| 536 | gaps=snapshot.gaps, |
| 537 | closing_reads=snapshot.closing_reads, |
| 538 | synthesis_paragraphs=synthesis_paragraphs, |
| 539 | ) |
| 540 | |
| 541 | |
| 542 | def load_month_snapshots( |
| 543 | content_root: Path, years: Iterable[int] | None = None |
| 544 | ) -> list[MonthSnapshot]: |
| 545 | if years: |
| 546 | paths = [] |
| 547 | for year in sorted(set(years)): |
| 548 | paths.extend(sorted((content_root / "monthly" / str(year)).glob("*.md"))) |
| 549 | else: |
| 550 | paths = sorted((content_root / "monthly").glob("*/*.md")) |
| 551 | snapshots = [ |
| 552 | load_month_snapshot_with_preference(path, content_root) for path in paths if path.is_file() |
| 553 | ] |
| 554 | return sorted(snapshots, key=lambda item: (item.year, item.month)) |
| 555 | |
| 556 | |
| 557 | def word_count(text: str) -> int: |
| 558 | return len(WORD_PATTERN.findall(text)) |
| 559 | |
| 560 | |
| 561 | def trim_words(text: str, limit: int) -> str: |
| 562 | words = text.split() |
| 563 | if len(words) <= limit: |
| 564 | return text.strip() |
| 565 | return " ".join(words[:limit]).rstrip(",;:.") + "…" |
| 566 | |
| 567 | |
| 568 | def compress_phrase(text: str, limit: int = 24) -> str: |
| 569 | cleaned = strip_markdown(text) |
| 570 | cleaned = re.sub(r"^(Week \d+\s+|W\d+\s+)", "", cleaned) |
| 571 | cleaned = re.sub( |
| 572 | r"^(The durable signal this week |This week |Week \d+ |W\d+ )", |
| 573 | "", |
| 574 | cleaned, |
| 575 | flags=re.IGNORECASE, |
| 576 | ) |
| 577 | cleaned = re.sub(r"\s+", " ", cleaned).strip().rstrip(".") |
| 578 | return trim_words(cleaned, limit) |
| 579 | |
| 580 | |
| 581 | def join_phrases(parts: Iterable[str], *, conjunction: str = "and") -> str: |
| 582 | items = [part.strip() for part in parts if part and part.strip()] |
| 583 | if not items: |
| 584 | return "" |
| 585 | if len(items) == 1: |
| 586 | return items[0] |
| 587 | if len(items) == 2: |
| 588 | return f"{items[0]} {conjunction} {items[1]}" |
| 589 | return f"{', '.join(items[:-1])}, {conjunction} {items[-1]}" |
| 590 | |
| 591 | |
| 592 | def keyword_score(text: str, keywords: Iterable[str]) -> int: |
| 593 | lowered = text.lower() |
| 594 | return sum(1 for keyword in keywords if keyword in lowered) |
| 595 | |
| 596 | |
| 597 | def detect_family_arc(months: list[MonthSnapshot], family: TrendFamily) -> list[str]: |
| 598 | stages: list[str] = [] |
| 599 | family_seen = False |
| 600 | for month in months: |
| 601 | lowered = month.text_blob.lower() |
| 602 | if keyword_score(lowered, family.keywords): |
| 603 | family_seen = True |
| 604 | for stage, keywords in family.stages: |
| 605 | if any(keyword in lowered for keyword in keywords) and stage not in stages: |
| 606 | stages.append(stage) |
| 607 | if family_seen and not stages: |
| 608 | stages.append("emerging") |
| 609 | return stages |
| 610 | |
| 611 | |
| 612 | def build_theme_sentence(year: int, arcs: dict[str, list[str]]) -> str: |
| 613 | has_skills = bool(arcs.get("agent-skills")) |
| 614 | has_noise = bool(arcs.get("platform-gaming")) |
| 615 | has_security = bool(arcs.get("security-gap")) |
| 616 | has_local = bool(arcs.get("self-hosted-ai")) |
| 617 | if has_skills and has_noise: |
| 618 | sentence = ( |
| 619 | f"{year} has been a split-screen story: agent tooling kept solidifying into a real distribution layer " |
| 620 | "while GitHub discovery got easier to game." |
| 621 | ) |
| 622 | elif has_skills: |
| 623 | sentence = f"{year} has mainly been the year agent tooling stopped looking experimental and started behaving like infrastructure." |
| 624 | else: |
| 625 | sentence = f"{year} has so far been defined less by single launches than by shifts in how the ecosystem is organizing itself." |
| 626 | if has_security: |
| 627 | sentence += " The ecosystem moved faster on capability than on trust." |
| 628 | elif has_local: |
| 629 | sentence += " Control, cost, and local execution kept gaining weight." |
| 630 | return sentence |
| 631 | |
| 632 | |
| 633 | def summarize_month(month: MonthSnapshot) -> str: |
| 634 | if month.synthesis_paragraphs: |
| 635 | return trim_words(strip_markdown(month.synthesis_paragraphs[0]), 40) |
| 636 | month_arcs = {family.key: detect_family_arc([month], family) for family in TREND_FAMILIES} |
| 637 | parts: list[str] = [] |
| 638 | |
| 639 | agent_arc = month_arcs.get("agent-skills", []) |
| 640 | if "globalization" in agent_arc and "verticalization" in agent_arc: |
| 641 | parts.append("agent skills globalized and started splitting into tighter verticals") |
| 642 | elif "economy" in agent_arc and "infrastructure" in agent_arc: |
| 643 | parts.append("agent skills hardened from plumbing into an economy") |
| 644 | elif "economy" in agent_arc: |
| 645 | parts.append("agent skills started looking like a real market layer") |
| 646 | elif "infrastructure" in agent_arc: |
| 647 | parts.append("agent tooling kept hardening into infrastructure") |
| 648 | |
| 649 | local_arc = month_arcs.get("self-hosted-ai", []) |
| 650 | if "local sovereignty" in local_arc: |
| 651 | parts.append("self-hosted and local-sovereignty tools gained real momentum") |
| 652 | elif "self-hosted workspaces" in local_arc: |
| 653 | parts.append("self-hosted AI workspaces became more credible") |
| 654 | |
| 655 | if month_arcs.get("security-gap"): |
| 656 | parts.append("the security gap stayed more visible than the fixes") |
| 657 | |
| 658 | platform_arc = month_arcs.get("platform-gaming", []) |
| 659 | if "fork-inflation" in platform_arc: |
| 660 | parts.append("fork inflation replaced the earlier star-farming playbook") |
| 661 | elif "star-farming" in platform_arc: |
| 662 | parts.append("coordinated star-farming made discovery harder to trust") |
| 663 | |
| 664 | if parts: |
| 665 | return ( |
| 666 | "; ".join(parts[:-1]) + ("" if len(parts) < 2 else "; ") + parts[-1] |
| 667 | if len(parts) > 1 |
| 668 | else parts[0] |
| 669 | ) |
| 670 | |
| 671 | source = ( |
| 672 | month.yearly_source_paragraphs[0] if month.yearly_source_paragraphs else month.text_blob |
| 673 | ) |
| 674 | return trim_words(strip_markdown(source), 32) |
| 675 | |
| 676 | |
| 677 | def build_month_bridge(month: MonthSnapshot, position: int, total: int) -> str: |
| 678 | if total == 1: |
| 679 | return f"{month.month_name} supplied the year's opening evidence" |
| 680 | if position == 0: |
| 681 | return f"{month.month_name} set the initial tone" |
| 682 | if position == total - 1: |
| 683 | return f"{month.month_name} pushed the story further" |
| 684 | return f"in {month.month_name}" |
| 685 | |
| 686 | |
| 687 | def build_opening_paragraph(months: list[MonthSnapshot], arcs: dict[str, list[str]]) -> str: |
| 688 | opening = build_theme_sentence(months[0].year, arcs) |
| 689 | span = ( |
| 690 | months[0].month_name |
| 691 | if len(months) == 1 |
| 692 | else f"From {months[0].month_name} through {months[-1].month_name}" |
| 693 | ) |
| 694 | durable_categories: list[str] = [] |
| 695 | if arcs.get("agent-skills"): |
| 696 | durable_categories.append("agent skills as a real distribution layer") |
| 697 | if arcs.get("self-hosted-ai"): |
| 698 | durable_categories.append("local and self-hosted execution as a durable buyer priority") |
| 699 | if arcs.get("security-gap"): |
| 700 | durable_categories.append("agent security as the main unresolved infrastructure gap") |
| 701 | if not durable_categories: |
| 702 | durable_categories.append("workflow-level shifts rather than one-off launches") |
| 703 | return ( |
| 704 | f"{opening} {span}, the important change was not a parade of isolated repositories but the way a few categories kept hardening: " |
| 705 | f"{join_phrases(durable_categories)}. The year so far reads less like a sequence of weekly surprises and more like an ecosystem choosing its operating model." |
| 706 | ) |
| 707 | |
| 708 | |
| 709 | def build_evolution_paragraph(months: list[MonthSnapshot]) -> str: |
| 710 | if not months: |
| 711 | return "" |
| 712 | fragments = [ |
| 713 | f"{build_month_bridge(month, index, len(months))} when {summarize_month(month).rstrip('.')}" |
| 714 | for index, month in enumerate(months) |
| 715 | ] |
| 716 | if len(fragments) == 1: |
| 717 | body = fragments[0] |
| 718 | else: |
| 719 | body = "; ".join(fragments[:-1]) + f"; {fragments[-1]}" |
| 720 | return ( |
| 721 | f"The monthly progression is clear: {body}. Taken together, those shifts show a market moving from experimentation toward packaging, distribution, and operating discipline. " |
| 722 | "Even when the surface story changes from one month to the next, the deeper motion is cumulative rather than episodic." |
| 723 | ) |
| 724 | |
| 725 | |
| 726 | def build_pattern_paragraph(arcs: dict[str, list[str]]) -> str: |
| 727 | sentences: list[str] = [] |
| 728 | agent_arc = arcs.get("agent-skills", []) |
| 729 | if agent_arc: |
| 730 | if "verticalization" in agent_arc or "globalization" in agent_arc: |
| 731 | sentences.append( |
| 732 | "The category that hardened fastest was agent skills: what began as infrastructure and workflow plumbing started behaving like a market, then spread into more specific geographies, languages, and job-shaped use cases." |
| 733 | ) |
| 734 | else: |
| 735 | sentences.append( |
| 736 | "The clearest durable category was agent skills, which stopped looking like a novelty and started looking like shared infrastructure." |
| 737 | ) |
| 738 | local_arc = arcs.get("self-hosted-ai", []) |
| 739 | if local_arc: |
| 740 | sentences.append( |
| 741 | "Self-hosted and local-first tooling also matured from a cost or billing workaround into a control story about sovereignty, reliability, and execution on hardware teams already own." |
| 742 | ) |
| 743 | platform_arc = arcs.get("platform-gaming", []) |
| 744 | if platform_arc: |
| 745 | sentences.append( |
| 746 | "The pattern that mutated instead of fading was platform gaming: the noise never really disappeared, it simply changed tactics from star-farming to fork inflation and then into more industrialized spam, fraud, and activator-style clutter." |
| 747 | ) |
| 748 | security_arc = arcs.get("security-gap", []) |
| 749 | if security_arc: |
| 750 | sentences.append( |
| 751 | "The prediction that capability would outrun trust was confirmed every month, because nothing in the visible tooling stack closed the gaps around agent isolation, prompt injection defense, or skills supply-chain auditing." |
| 752 | ) |
| 753 | return " ".join(sentences) |
| 754 | |
| 755 | |
| 756 | def build_prediction_review(arcs: dict[str, list[str]]) -> str: |
| 757 | confirmations: list[str] = [] |
| 758 | if "globalization" in arcs.get("agent-skills", []): |
| 759 | confirmations.append("skills did globalize") |
| 760 | if "verticalization" in arcs.get("agent-skills", []): |
| 761 | confirmations.append("skills also verticalized quickly") |
| 762 | if len(arcs.get("platform-gaming", [])) >= 2: |
| 763 | confirmations.append("discovery-layer abuse mutated instead of self-correcting") |
| 764 | if arcs.get("self-hosted-ai"): |
| 765 | confirmations.append( |
| 766 | "local and self-hosted AI kept becoming a category rather than a workaround" |
| 767 | ) |
| 768 | if arcs.get("security-gap"): |
| 769 | confirmations.append("the trust and security gap remained open") |
| 770 | weakened: list[str] = [] |
| 771 | if arcs.get("platform-gaming"): |
| 772 | weakened.append("the hope that GitHub discovery noise would self-correct") |
| 773 | if arcs.get("security-gap"): |
| 774 | weakened.append("the idea that trust tooling would catch up on its own") |
| 775 | if "verticalization" in arcs.get("agent-skills", []): |
| 776 | weakened.append( |
| 777 | "the simpler thesis that one general-purpose agent workflow would dominate everything" |
| 778 | ) |
| 779 | if not confirmations and not weakened: |
| 780 | return "The running predictions stayed directionally useful: the biggest structural questions still look unresolved." |
| 781 | sentences: list[str] = [] |
| 782 | if confirmations: |
| 783 | sentences.append(f"What was confirmed: {join_phrases(confirmations)}.") |
| 784 | if weakened: |
| 785 | sentences.append(f"What weakened: {join_phrases(weakened)}.") |
| 786 | sentences.append( |
| 787 | "That leaves the main story of the year intact: builders are getting more serious about packaging and operating agents, while the trust, filtering, and governance layers remain conspicuously behind." |
| 788 | ) |
| 789 | return " ".join(sentences) |
| 790 | |
| 791 | |
| 792 | def compress_narrative(paragraphs: list[str], max_words: int = 500) -> str: |
| 793 | text = "\n\n".join(paragraph.strip() for paragraph in paragraphs if paragraph.strip()) |
| 794 | if word_count(text) <= max_words: |
| 795 | return text |
| 796 | compressed = text |
| 797 | for limit in (480, 460, 430, 400, 360): |
| 798 | words = compressed.split() |
| 799 | if len(words) <= max_words: |
| 800 | break |
| 801 | compressed = " ".join(words[:limit]).rstrip(",;:.") + "…" |
| 802 | return compressed |
| 803 | |
| 804 | |
| 805 | def synthesize_year(months: list[MonthSnapshot]) -> str: |
| 806 | arcs = {family.key: detect_family_arc(months, family) for family in TREND_FAMILIES} |
| 807 | paragraphs = [ |
| 808 | build_opening_paragraph(months, arcs), |
| 809 | build_evolution_paragraph(months), |
| 810 | build_pattern_paragraph(arcs), |
| 811 | build_prediction_review(arcs), |
| 812 | ] |
| 813 | return compress_narrative(paragraphs) |
| 814 | |
| 815 | |
| 816 | def generate_yearly_title(year: int, arcs: dict[str, list[str]]) -> str: |
| 817 | """Generate an SEO-friendly editorial title (max 70 chars) for yearly narrative.""" |
| 818 | has_skills = bool(arcs.get("agent-skills")) |
| 819 | has_security = bool(arcs.get("security-gap")) |
| 820 | has_noise = bool(arcs.get("platform-gaming")) |
| 821 | has_local = bool(arcs.get("self-hosted-ai")) |
| 822 | |
| 823 | if has_skills and has_security: |
| 824 | title = f"When Agents Became Infrastructure — {year} So Far" |
| 825 | elif has_skills and has_noise: |
| 826 | title = f"Agents Rise While Discovery Noise Mutates — {year}" |
| 827 | elif has_skills: |
| 828 | title = f"Agent Tooling Grows Up — {year} So Far" |
| 829 | elif has_security: |
| 830 | title = f"Capability Outpaces Trust — {year} So Far" |
| 831 | elif has_local: |
| 832 | title = f"Local AI Takes Hold — {year} So Far" |
| 833 | else: |
| 834 | title = f"The Ecosystem Reorganizes — {year} So Far" |
| 835 | |
| 836 | return title[:70] |
| 837 | |
| 838 | |
| 839 | def _extract_summary(narrative: str, max_length: int = 155) -> str: |
| 840 | """Extract a ≤155-character summary from the narrative's first sentence.""" |
| 841 | narrative = narrative.strip() |
| 842 | # Take first sentence (up to first . ! or ? followed by whitespace or end) |
| 843 | first_sentence_match = re.match(r"(.+?[.!?])(?:\s|$)", narrative) |
| 844 | if first_sentence_match: |
| 845 | sentence = first_sentence_match.group(1).strip() |
| 846 | if len(sentence) <= max_length: |
| 847 | return sentence |
| 848 | # Truncate at last word boundary within limit |
| 849 | truncated = sentence[: max_length - 1].rsplit(" ", 1)[0] |
| 850 | return truncated.rstrip(".,;:") + "…" |
| 851 | # Fallback: truncate narrative at word boundary |
| 852 | if len(narrative) <= max_length: |
| 853 | return narrative.strip() |
| 854 | truncated = narrative[: max_length - 1].rsplit(" ", 1)[0] |
| 855 | return truncated.rstrip(".,;:") + "…" |
| 856 | |
| 857 | |
| 858 | def build_yearly_narrative_pages( |
| 859 | content_root: Path, years: Iterable[int] | None = None |
| 860 | ) -> list[YearlyNarrativePage]: |
| 861 | grouped: dict[int, list[MonthSnapshot]] = {} |
| 862 | for snapshot in load_month_snapshots(content_root, years): |
| 863 | grouped.setdefault(snapshot.year, []).append(snapshot) |
| 864 | |
| 865 | pages: list[YearlyNarrativePage] = [] |
| 866 | for year, months in sorted(grouped.items()): |
| 867 | ordered = sorted(months, key=lambda item: item.month) |
| 868 | narrative = synthesize_year(ordered) |
| 869 | arcs = {family.key: detect_family_arc(ordered, family) for family in TREND_FAMILIES} |
| 870 | title = generate_yearly_title(year, arcs) |
| 871 | summary = _extract_summary(narrative) |
| 872 | month_slugs = [month.month_slug for month in ordered] |
| 873 | nav_links = [] |
| 874 | for slug in month_slugs: |
| 875 | parts = slug.split("-") |
| 876 | if len(parts) == 2: |
| 877 | year_str, month_str = parts |
| 878 | month_num = int(month_str) |
| 879 | month_name = MONTH_NAMES.get(month_num, month_str) |
| 880 | nav_links.append(f"[{month_name}](/monthly/{year_str}/{month_str}/)") |
| 881 | nav_prefix = "" |
| 882 | if nav_links: |
| 883 | nav_prefix = f"**Monthly reports:** {' · '.join(nav_links)}\n\n" |
| 884 | pages.append( |
| 885 | YearlyNarrativePage( |
| 886 | year=year, |
| 887 | path=content_root / "yearly" / f"{year}.md", |
| 888 | frontmatter={ |
| 889 | "title": title, |
| 890 | "date": ordered[-1].date, |
| 891 | "year": year, |
| 892 | "categories": ["yearly"], |
| 893 | "months_covered": month_slugs, |
| 894 | "format": "narrative", |
| 895 | "summary": summary, |
| 896 | }, |
| 897 | narrative=nav_prefix + narrative, |
| 898 | ) |
| 899 | ) |
| 900 | return pages |
| 901 | |
| 902 | |
| 903 | def render_yearly_page(page: YearlyNarrativePage) -> str: |
| 904 | body = f"## Year in Review\n\n{page.narrative}\n" |
| 905 | return render_frontmatter(page.frontmatter) + body |
| 906 | |
| 907 | |
| 908 | def generate_yearly_narratives( |
| 909 | content_root: Path, years: Iterable[int] | None = None |
| 910 | ) -> list[Path]: |
| 911 | written: list[Path] = [] |
| 912 | for page in build_yearly_narrative_pages(content_root, years): |
| 913 | page.path.parent.mkdir(parents=True, exist_ok=True) |
| 914 | page.path.write_text(render_yearly_page(page), encoding="utf-8") |
| 915 | written.append(page.path) |
| 916 | return written |
| 917 | |
| 918 | |
| 919 | def main(argv: list[str] | None = None) -> int: |
| 920 | args = parse_args(argv) |
| 921 | written = generate_yearly_narratives(args.content_root, args.years) |
| 922 | if not written: |
| 923 | print(f"No monthly rollups found under {args.content_root / 'monthly'}") |
| 924 | return 0 |
| 925 | for path in written: |
| 926 | print(f"Generated {path}") |
| 927 | return 0 |
| 928 | |
| 929 | |
| 930 | if __name__ == "__main__": |
| 931 | raise SystemExit(main()) |