| 1 | """Month synthesis: deterministic monthly narrative generator. |
| 2 | |
| 3 | Consumes weekly analysis summaries for a given month and produces a synthesis |
| 4 | artifact with: |
| 5 | - Structured frontmatter: title, summary, themes, persistent/accelerating/weakening |
| 6 | themes, key_gaps, top_repos, source_checksum, weeks_covered |
| 7 | - Body sections: Month Synthesis (narrative), Weekly Reports (cross-linked list), |
| 8 | Trend Arc (persistent/accelerating/weakening theme bullets), Prediction Review |
| 9 | - Summary field (≤28 words) suitable for SEO meta descriptions |
| 10 | - Source checksum for idempotent regeneration (skips if input unchanged) |
| 11 | """ |
| 12 | |
| 13 | import hashlib |
| 14 | import json |
| 15 | import re |
| 16 | from collections import Counter |
| 17 | from dataclasses import dataclass, replace |
| 18 | from pathlib import Path |
| 19 | from typing import Any |
| 20 | |
| 21 | import scripts.analysis_gate as analysis_gate |
| 22 | |
| 23 | MONTH_NAMES = { |
| 24 | 1: "January", |
| 25 | 2: "February", |
| 26 | 3: "March", |
| 27 | 4: "April", |
| 28 | 5: "May", |
| 29 | 6: "June", |
| 30 | 7: "July", |
| 31 | 8: "August", |
| 32 | 9: "September", |
| 33 | 10: "October", |
| 34 | 11: "November", |
| 35 | 12: "December", |
| 36 | } |
| 37 | |
| 38 | SECTION_PATTERN = re.compile(r"(?m)^##\s+(.+?)\s*$") |
| 39 | WORD_PATTERN = re.compile(r"\S+") |
| 40 | SYNTHESIS_VERSION = 2 |
| 41 | |
| 42 | |
| 43 | @dataclass(frozen=True) |
| 44 | class MonthSynthesis: |
| 45 | path: Path |
| 46 | year: int |
| 47 | month: int |
| 48 | date: str |
| 49 | weeks_covered: tuple[str, ...] |
| 50 | summary: str |
| 51 | narrative: str |
| 52 | trend_arc: str |
| 53 | prediction_review: str |
| 54 | weekly_reports: tuple[str, ...] |
| 55 | themes: tuple[str, ...] |
| 56 | persistent_themes: tuple[str, ...] |
| 57 | accelerating_themes: tuple[str, ...] |
| 58 | weakening_themes: tuple[str, ...] |
| 59 | key_gaps: tuple[str, ...] |
| 60 | top_repos: tuple[str, ...] |
| 61 | source_checksum: str |
| 62 | status: str = "generated" |
| 63 | |
| 64 | @property |
| 65 | def month_slug(self) -> str: |
| 66 | return f"{self.year}-{self.month:02d}" |
| 67 | |
| 68 | @property |
| 69 | def title(self) -> str: |
| 70 | return f"{MONTH_NAMES[self.month]} {self.year} Month Synthesis" |
| 71 | |
| 72 | |
| 73 | def yaml_quote(value: str) -> str: |
| 74 | return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' |
| 75 | |
| 76 | |
| 77 | def yaml_value(value: Any) -> str: |
| 78 | if isinstance(value, str): |
| 79 | return yaml_quote(value) |
| 80 | if isinstance(value, bool): |
| 81 | return "true" if value else "false" |
| 82 | if isinstance(value, int): |
| 83 | return str(value) |
| 84 | if isinstance(value, (list, tuple)): |
| 85 | return f"[{', '.join(yaml_value(item) for item in value)}]" |
| 86 | return str(value) |
| 87 | |
| 88 | |
| 89 | def render_frontmatter(frontmatter: dict[str, Any]) -> str: |
| 90 | lines = ["---"] |
| 91 | for key, value in frontmatter.items(): |
| 92 | lines.append(f"{key}: {yaml_value(value)}") |
| 93 | lines.extend(["---", "", ""]) |
| 94 | return "\n".join(lines) |
| 95 | |
| 96 | |
| 97 | def split_sections(body: str) -> dict[str, str]: |
| 98 | matches = list(SECTION_PATTERN.finditer(body)) |
| 99 | sections: dict[str, str] = {} |
| 100 | for index, match in enumerate(matches): |
| 101 | start = match.end() |
| 102 | end = matches[index + 1].start() if index + 1 < len(matches) else len(body) |
| 103 | sections[match.group(1).strip()] = body[start:end].strip("\n") |
| 104 | return sections |
| 105 | |
| 106 | |
| 107 | def normalize_text(value: str) -> str: |
| 108 | return re.sub(r"\s+", " ", value.strip()) |
| 109 | |
| 110 | |
| 111 | def strip_markdown(value: str) -> str: |
| 112 | cleaned = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", value) |
| 113 | cleaned = cleaned.replace("**", "").replace("*", "").replace("`", "") |
| 114 | return normalize_text(cleaned) |
| 115 | |
| 116 | |
| 117 | def trim_words(text: str, limit: int) -> str: |
| 118 | words = text.split() |
| 119 | if len(words) <= limit: |
| 120 | return text.strip() |
| 121 | return " ".join(words[:limit]).rstrip(",;:.") + "…" |
| 122 | |
| 123 | |
| 124 | def dedupe(values: list[str]) -> list[str]: |
| 125 | result: list[str] = [] |
| 126 | seen: set[str] = set() |
| 127 | for value in values: |
| 128 | cleaned = value.strip() |
| 129 | if not cleaned or cleaned in seen: |
| 130 | continue |
| 131 | seen.add(cleaned) |
| 132 | result.append(cleaned) |
| 133 | return result |
| 134 | |
| 135 | |
| 136 | def tag_label(tag: str) -> str: |
| 137 | return tag.replace("-", " ") |
| 138 | |
| 139 | |
| 140 | def join_terms(values: list[str]) -> str: |
| 141 | if not values: |
| 142 | return "" |
| 143 | if len(values) == 1: |
| 144 | return values[0] |
| 145 | if len(values) == 2: |
| 146 | return f"{values[0]} and {values[1]}" |
| 147 | return f"{', '.join(values[:-1])}, and {values[-1]}" |
| 148 | |
| 149 | |
| 150 | def top_sentences(values: list[str], *, limit: int = 2, words: int = 18) -> list[str]: |
| 151 | sentences: list[str] = [] |
| 152 | for value in dedupe([normalize_text(value) for value in values]): |
| 153 | if not value: |
| 154 | continue |
| 155 | sentences.append(trim_words(strip_markdown(value), words).rstrip(".")) |
| 156 | if len(sentences) >= limit: |
| 157 | break |
| 158 | return sentences |
| 159 | |
| 160 | |
| 161 | def build_weekly_reports(items: list[Any]) -> tuple[str, ...]: |
| 162 | return tuple( |
| 163 | f"- [{item.week_title}]({item.week_link}) — {trim_words(strip_markdown(item.summary), 18)}" |
| 164 | for item in items |
| 165 | ) |
| 166 | |
| 167 | |
| 168 | def compress_week(item: Any) -> dict[str, Any]: |
| 169 | return { |
| 170 | "week": item.week, |
| 171 | "title": item.title, |
| 172 | "summary": item.summary, |
| 173 | "top_repo": item.top_repo, |
| 174 | "tags": list(item.tags), |
| 175 | "signal": item.signal, |
| 176 | "noise": item.noise, |
| 177 | "gaps": item.gaps, |
| 178 | "conclusion": item.conclusion, |
| 179 | "featured_repos": list(item.featured_repos[:5]), |
| 180 | } |
| 181 | |
| 182 | |
| 183 | def build_month_synthesis_pack(items: list[Any]) -> str: |
| 184 | payload = { |
| 185 | "synthesis_version": SYNTHESIS_VERSION, |
| 186 | "month": f"{items[0].year}-{items[0].month:02d}", |
| 187 | "weeks_covered": [item.week for item in items], |
| 188 | "weeks": [compress_week(item) for item in items], |
| 189 | } |
| 190 | return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) |
| 191 | |
| 192 | |
| 193 | def source_checksum(pack: str) -> str: |
| 194 | return "sha256:" + hashlib.sha256(pack.encode("utf-8")).hexdigest() |
| 195 | |
| 196 | |
| 197 | def synthesis_path(analyzed_dir: Path, year: int, month: int) -> Path: |
| 198 | return analyzed_dir / f"{year}-{month:02d}-month-synthesis.md" |
| 199 | |
| 200 | |
| 201 | def _theme_trajectory( |
| 202 | items: list[Any], |
| 203 | ) -> tuple[list[str], list[str], list[str], list[str], list[str]]: |
| 204 | tag_counts = Counter(tag for item in items for tag in set(item.tags)) |
| 205 | weeks_per_tag: dict[str, list[int]] = {} |
| 206 | if len(items) == 1: |
| 207 | ordered_themes = [tag for tag, _ in tag_counts.most_common(5)] |
| 208 | return ordered_themes, [], [], [], [] |
| 209 | midpoint = max(1, len(items) // 2) |
| 210 | for index, item in enumerate(items): |
| 211 | for tag in set(item.tags): |
| 212 | weeks_per_tag.setdefault(tag, []).append(index) |
| 213 | |
| 214 | persistent: list[str] = [] |
| 215 | accelerating: list[str] = [] |
| 216 | weakening: list[str] = [] |
| 217 | emerging: list[str] = [] |
| 218 | for tag, positions in weeks_per_tag.items(): |
| 219 | in_first_half = any(position < midpoint for position in positions) |
| 220 | in_second_half = any(position >= midpoint for position in positions) |
| 221 | if len(positions) >= 2: |
| 222 | persistent.append(tag) |
| 223 | if in_second_half and not in_first_half: |
| 224 | emerging.append(tag) |
| 225 | elif in_first_half and not in_second_half: |
| 226 | weakening.append(tag) |
| 227 | elif ( |
| 228 | positions |
| 229 | and positions[-1] >= midpoint |
| 230 | and positions[0] < midpoint |
| 231 | and len(positions) >= 2 |
| 232 | ): |
| 233 | accelerating.append(tag) |
| 234 | |
| 235 | ordered_themes = [tag for tag, _ in tag_counts.most_common(5)] |
| 236 | persistent.sort(key=lambda tag: (-tag_counts[tag], tag)) |
| 237 | accelerating.sort(key=lambda tag: (-tag_counts[tag], tag)) |
| 238 | weakening.sort(key=lambda tag: (-tag_counts[tag], tag)) |
| 239 | emerging.sort(key=lambda tag: (-tag_counts[tag], tag)) |
| 240 | return ordered_themes, persistent, accelerating, weakening, emerging |
| 241 | |
| 242 | |
| 243 | def _word_count(text: str) -> int: |
| 244 | return len(WORD_PATTERN.findall(text)) |
| 245 | |
| 246 | |
| 247 | def _trim_to_range(text: str, *, minimum: int = 200, maximum: int = 350) -> str: |
| 248 | cleaned = "\n\n".join(part.strip() for part in text.split("\n\n") if part.strip()) |
| 249 | count = _word_count(cleaned) |
| 250 | if count <= maximum: |
| 251 | return cleaned |
| 252 | words = cleaned.split() |
| 253 | trimmed = " ".join(words[:maximum]).rstrip(",;:.") + "…" |
| 254 | if _word_count(trimmed) >= minimum: |
| 255 | return trimmed |
| 256 | return cleaned |
| 257 | |
| 258 | |
| 259 | def synthesize_month( |
| 260 | items: list[Any], analyzed_dir: Path, checksum: str | None = None |
| 261 | ) -> MonthSynthesis: |
| 262 | if not items: |
| 263 | raise ValueError("Cannot synthesize an empty month") |
| 264 | |
| 265 | year = items[0].year |
| 266 | month = items[0].month |
| 267 | pack = build_month_synthesis_pack(items) |
| 268 | digest = checksum or source_checksum(pack) |
| 269 | path = synthesis_path(analyzed_dir, year, month) |
| 270 | |
| 271 | themes, persistent, accelerating, weakening, emerging = _theme_trajectory(items) |
| 272 | theme_labels = [tag_label(tag) for tag in themes[:3]] |
| 273 | persistent_labels = [tag_label(tag) for tag in persistent[:3]] |
| 274 | accelerating_labels = [tag_label(tag) for tag in (emerging + accelerating)[:3]] |
| 275 | weakening_labels = [tag_label(tag) for tag in weakening[:3]] |
| 276 | |
| 277 | summaries = [trim_words(strip_markdown(item.summary), 24) for item in items if item.summary] |
| 278 | signals = top_sentences([item.signal for item in items if item.signal], limit=2, words=22) |
| 279 | noise = top_sentences([item.noise for item in items if item.noise], limit=2, words=18) |
| 280 | gaps = top_sentences([item.gaps for item in items if item.gaps], limit=3, words=18) |
| 281 | conclusions = top_sentences( |
| 282 | [item.conclusion for item in items if item.conclusion], limit=2, words=20 |
| 283 | ) |
| 284 | top_repos = dedupe([item.top_repo for item in items if item.top_repo])[:4] |
| 285 | |
| 286 | summary = f"{MONTH_NAMES[month]} {year} was defined by {join_terms(theme_labels) if theme_labels else 'cross-week trend consolidation'}." |
| 287 | if accelerating_labels: |
| 288 | summary += f" Later in the month, {join_terms(accelerating_labels)} gathered pace." |
| 289 | elif noise: |
| 290 | summary += " The noise floor kept mutating instead of clearing." |
| 291 | summary = trim_words(summary, 28) |
| 292 | |
| 293 | opening = ( |
| 294 | f"{MONTH_NAMES[month]} {year} reads less like three isolated weekly spikes and more like one continuous adjustment in priorities. " |
| 295 | f"The month opened with {summaries[0] if summaries else 'a broad platform reset'} and ended with " |
| 296 | f"{summaries[-1] if summaries else 'a clearer hierarchy of durable themes'}, which means the center of gravity shifted without abandoning the strongest earlier signals." |
| 297 | ) |
| 298 | |
| 299 | theme_sentence_parts: list[str] = [] |
| 300 | if persistent_labels: |
| 301 | theme_sentence_parts.append( |
| 302 | f"Persistent themes such as {join_terms(persistent_labels)} stayed present across multiple weeks" |
| 303 | ) |
| 304 | if accelerating_labels: |
| 305 | theme_sentence_parts.append( |
| 306 | f"Later reports pushed {join_terms(accelerating_labels)} from interesting side threads into defining narratives" |
| 307 | ) |
| 308 | if weakening_labels: |
| 309 | theme_sentence_parts.append( |
| 310 | f"Early-month concerns around {join_terms(weakening_labels)} faded relative to the stronger follow-on trends" |
| 311 | ) |
| 312 | if len(top_repos) > 1: |
| 313 | theme_sentence_parts.append( |
| 314 | f"The month's anchor repos moved from {join_terms(top_repos[:2])} toward " |
| 315 | f"{top_repos[-1]}, reinforcing that the winning projects were the ones narrowing scope while deepening practical utility" |
| 316 | ) |
| 317 | elif top_repos: |
| 318 | theme_sentence_parts.append( |
| 319 | f"{top_repos[0]} served as the clearest anchor repo, which fits a month where practical utility mattered more than novelty alone" |
| 320 | ) |
| 321 | theme_paragraph = ". ".join(part.rstrip(".") for part in theme_sentence_parts if part) + "." |
| 322 | |
| 323 | signal_paragraph = ( |
| 324 | f"The cross-week signal strengthened around {'; '.join(signals) if signals else 'operationally useful work rather than one-off hype'}. " |
| 325 | f"At the same time, the month never solved its trust problem: " |
| 326 | f"{'; '.join(gaps) if gaps else 'the same defensive gaps kept resurfacing'}." |
| 327 | ) |
| 328 | |
| 329 | prediction_sentence = "Most weekly predictions held up" |
| 330 | if weakening_labels: |
| 331 | prediction_sentence += f": the month kept validating {join_terms(accelerating_labels or persistent_labels or theme_labels)} while {join_terms(weakening_labels)} lost urgency" |
| 332 | elif accelerating_labels or persistent_labels: |
| 333 | prediction_sentence += f": later weeks reinforced {join_terms(accelerating_labels or persistent_labels)} instead of reversing them" |
| 334 | else: |
| 335 | prediction_sentence += ( |
| 336 | ": the later reports mostly confirmed the earlier direction of travel" |
| 337 | ) |
| 338 | if conclusions: |
| 339 | prediction_sentence += f". In retrospect, the clearest forward-looking reads were that {'; '.join(conclusions)}." |
| 340 | else: |
| 341 | prediction_sentence += "." |
| 342 | |
| 343 | if noise: |
| 344 | prediction_sentence += f" The main counter-signal was noise that evolved from {' to '.join(noise[:2]) if len(noise) > 1 else noise[0]}." |
| 345 | |
| 346 | narrative = _trim_to_range( |
| 347 | "\n\n".join([opening, theme_paragraph, signal_paragraph, prediction_sentence]) |
| 348 | ) |
| 349 | |
| 350 | trend_arc_lines = [ |
| 351 | f"- Persistent themes: {join_terms(persistent_labels) if persistent_labels else 'none yet'}.", |
| 352 | f"- Accelerating themes: {join_terms(accelerating_labels) if accelerating_labels else 'none yet'}.", |
| 353 | f"- Weakened or receding themes: {join_terms(weakening_labels) if weakening_labels else 'none clearly receding yet'}.", |
| 354 | ] |
| 355 | if top_repos: |
| 356 | trend_arc_lines.append(f"- Top repos that anchored the month: {join_terms(top_repos)}.") |
| 357 | |
| 358 | prediction_lines = [prediction_sentence] |
| 359 | if gaps: |
| 360 | prediction_lines.append( |
| 361 | "The biggest unresolved gaps remained " |
| 362 | + f"{join_terms(gaps[:3])}, so the monthly story still points to missing trust, filtering, or operational scaffolding." |
| 363 | ) |
| 364 | |
| 365 | return MonthSynthesis( |
| 366 | path=path, |
| 367 | year=year, |
| 368 | month=month, |
| 369 | date=items[-1].date.isoformat(), |
| 370 | weeks_covered=tuple(item.week for item in items), |
| 371 | summary=summary, |
| 372 | narrative=narrative, |
| 373 | trend_arc="\n".join(trend_arc_lines), |
| 374 | prediction_review="\n\n".join(prediction_lines), |
| 375 | weekly_reports=build_weekly_reports(items), |
| 376 | themes=tuple(themes), |
| 377 | persistent_themes=tuple(persistent), |
| 378 | accelerating_themes=tuple(dedupe(emerging + accelerating)), |
| 379 | weakening_themes=tuple(weakening), |
| 380 | key_gaps=tuple(gaps), |
| 381 | top_repos=tuple(top_repos), |
| 382 | source_checksum=digest, |
| 383 | ) |
| 384 | |
| 385 | |
| 386 | def render_month_synthesis(synthesis: MonthSynthesis) -> str: |
| 387 | frontmatter = { |
| 388 | "title": synthesis.title, |
| 389 | "date": synthesis.date, |
| 390 | "month": synthesis.month_slug, |
| 391 | "weeks_covered": list(synthesis.weeks_covered), |
| 392 | "categories": ["monthly-synthesis"], |
| 393 | "summary": synthesis.summary, |
| 394 | "status": synthesis.status, |
| 395 | "source_checksum": synthesis.source_checksum, |
| 396 | "themes": list(synthesis.themes), |
| 397 | "persistent_themes": list(synthesis.persistent_themes), |
| 398 | "accelerating_themes": list(synthesis.accelerating_themes), |
| 399 | "weakening_themes": list(synthesis.weakening_themes), |
| 400 | "key_gaps": list(synthesis.key_gaps), |
| 401 | "top_repos": list(synthesis.top_repos), |
| 402 | } |
| 403 | body = ( |
| 404 | f"## Month Synthesis\n\n{synthesis.narrative}\n\n" |
| 405 | f"## Weekly Reports\n\n" + "\n".join(synthesis.weekly_reports) + "\n\n" |
| 406 | f"## Trend Arc\n\n{synthesis.trend_arc}\n\n" |
| 407 | f"## Prediction Review\n\n{synthesis.prediction_review}\n" |
| 408 | ) |
| 409 | return render_frontmatter(frontmatter) + body |
| 410 | |
| 411 | |
| 412 | def write_month_synthesis(synthesis: MonthSynthesis) -> None: |
| 413 | synthesis.path.parent.mkdir(parents=True, exist_ok=True) |
| 414 | synthesis.path.write_text(render_month_synthesis(synthesis), encoding="utf-8") |
| 415 | |
| 416 | |
| 417 | def _frontmatter_list(frontmatter: dict[str, Any], key: str) -> tuple[str, ...]: |
| 418 | raw = frontmatter.get(key, []) |
| 419 | if isinstance(raw, list): |
| 420 | return tuple(str(item) for item in raw) |
| 421 | return () |
| 422 | |
| 423 | |
| 424 | def load_month_synthesis(path: Path) -> MonthSynthesis: |
| 425 | frontmatter, body = analysis_gate.extract_frontmatter(path.read_text(encoding="utf-8")) |
| 426 | month_slug = str(frontmatter["month"]) |
| 427 | year_text, month_text = month_slug.split("-", 1) |
| 428 | sections = split_sections(body) |
| 429 | weekly_reports = tuple( |
| 430 | line |
| 431 | for line in sections.get("Weekly Reports", "").splitlines() |
| 432 | if line.strip().startswith("- ") |
| 433 | ) |
| 434 | return MonthSynthesis( |
| 435 | path=path, |
| 436 | year=int(year_text), |
| 437 | month=int(month_text), |
| 438 | date=str(frontmatter["date"]), |
| 439 | weeks_covered=_frontmatter_list(frontmatter, "weeks_covered"), |
| 440 | summary=str(frontmatter.get("summary", "")), |
| 441 | narrative=sections.get("Month Synthesis", "").strip(), |
| 442 | trend_arc=sections.get("Trend Arc", "").strip(), |
| 443 | prediction_review=sections.get("Prediction Review", "").strip(), |
| 444 | weekly_reports=weekly_reports, |
| 445 | themes=_frontmatter_list(frontmatter, "themes"), |
| 446 | persistent_themes=_frontmatter_list(frontmatter, "persistent_themes"), |
| 447 | accelerating_themes=_frontmatter_list(frontmatter, "accelerating_themes"), |
| 448 | weakening_themes=_frontmatter_list(frontmatter, "weakening_themes"), |
| 449 | key_gaps=_frontmatter_list(frontmatter, "key_gaps"), |
| 450 | top_repos=_frontmatter_list(frontmatter, "top_repos"), |
| 451 | source_checksum=str(frontmatter.get("source_checksum", "")), |
| 452 | status=str(frontmatter.get("status", "generated")), |
| 453 | ) |
| 454 | |
| 455 | |
| 456 | def ensure_month_synthesis(items: list[Any], analyzed_dir: Path) -> MonthSynthesis: |
| 457 | if not items: |
| 458 | raise ValueError("Cannot synthesize an empty month") |
| 459 | pack = build_month_synthesis_pack(items) |
| 460 | checksum = source_checksum(pack) |
| 461 | path = synthesis_path(analyzed_dir, items[0].year, items[0].month) |
| 462 | if path.exists(): |
| 463 | cached = load_month_synthesis(path) |
| 464 | if ( |
| 465 | cached.weeks_covered == tuple(item.week for item in items) |
| 466 | and cached.source_checksum == checksum |
| 467 | ): |
| 468 | return replace(cached, weekly_reports=build_weekly_reports(items)) |
| 469 | synthesis = synthesize_month(items, analyzed_dir, checksum) |
| 470 | write_month_synthesis(synthesis) |
| 471 | return synthesis |