feat: divergence analysis — identify gaps between press and dev activity (#131)
feat: add divergence analysis - uncovered tech trends and unpublicized dev activity Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 19, 2026 at 20:17 UTC
0f8d4180c29b2fb3dbca7d3f3603a529fffa4255
3 files changed
+161
.squad/agents/farnsworth/history.md
+1
@@ -26,3 +26,4 @@
26
- **2026-05-19T15:08:00Z:** Leela milestone decomposition complete. Issues assigned to v0.5–v0.9 milestones. Scribe logged orchestration and merged decision. Your assigned v0.5 analysis and synthesis issues are ready. See `.squad/orchestration-log/2026-05-19T15-08-leela.md` for full decomposition outcome.
27
- **2026-05-19T15:22:00+02:00:** Topic-aware prompt template implemented (Issue #63). Key architecture decisions: (1) Used `{{#IF_TOPIC}}`/`{{#IF_NO_TOPIC}}` conditional blocks rather than Jinja2 to keep the template readable as standalone markdown and avoid adding template engine dependencies. (2) Wisdom injection is two-tier — global wisdom from `.squad/identity/wisdom.md` (existing) plus per-topic wisdom from `topics/{id}/wisdom.md` (new). (3) Render script (`scripts/render_topic_prompt.py`) is zero-dependency (stdlib only, with optional PyYAML), so it works in any CI environment without pip install. (4) Backward compatibility guaranteed: when no `squadscope.topic.yml` exists, the template collapses cleanly to general-mode analysis identical to the existing `analyze-weekly.md` behavior.
28
- **2026-05-19T20:07:19+02:00:** Fixed correlator "0 repos" bug (PR #130). Root cause: `correlate.py` loaded repos via `raw_data.get("repos")` but `crawl.py` writes them under `new_repos` and `trending_repos`. Key paths: `scripts/correlate.py:320`, `scripts/crawl.py:857-858`. Lesson: when integrating scripts in a pipeline, always verify the producer's *actual output schema* against the consumer's expected input schema — don't assume key names match. The CI skill pattern ("test the wire") would have caught this if applied at integration time.
29
+- **2026-05-19T20:13:00+02:00:** Divergence analysis implemented (PR #131). Architecture: `detect_divergences()` in `correlate.py` takes the correlation results and inverts them — unmatched articles become "uncovered tech trends", unmatched repos become "unpublicized dev activity". Articles grouped by first category/entity, repos by first topic. Renderer (`render_press_context.py`) appends divergences as a labeled section after the template. Key insight: divergence detection is cheap once correlations are computed — it's just set-difference on matched URLs and repo names. The editorial value is in surfacing *what's missing*, not just what aligns.
scripts/correlate.py
+102
@@ -212,6 +212,102 @@ def correlate_repo(repo: dict[str, Any], articles: list[dict[str, Any]]) -> dict
212
}
213
214
215
+def _extract_article_topic(article: dict[str, Any]) -> str:
216
+ """Extract a representative topic string from an article."""
217
+ categories = article.get("categories", [])
218
+ if categories:
219
+ return categories[0]
220
+ entities = article.get("entities", [])
221
+ if entities:
222
+ return entities[0]
223
+ title = article.get("title", "")
224
+ # Use first few meaningful words from title as fallback
225
+ words = [w for w in re.split(r"\s+", title) if len(w) > 3]
226
+ return " ".join(words[:3]) if words else "unknown"
227
+
228
+
229
+def _extract_repo_topic(repo: dict[str, Any]) -> str:
230
+ """Extract a representative topic string from a repo."""
231
+ topics = repo.get("topics", [])
232
+ if topics:
233
+ return topics[0]
234
+ description = repo.get("description") or ""
235
+ words = [w for w in re.split(r"\s+", description) if len(w) > 3]
236
+ return " ".join(words[:3]) if words else repo.get("name", "unknown")
237
+
238
+
239
+def detect_divergences(
240
+ repos: list[dict[str, Any]],
241
+ articles: list[dict[str, Any]],
242
+ correlations: list[dict[str, Any]],
243
+) -> dict[str, Any]:
244
+ """Detect divergences — gaps between press coverage and dev activity.
245
+
246
+ Returns two lists:
247
+ - uncovered_tech_trends: articles/topics with no matching GitHub activity
248
+ - unpublicized_dev_activity: repos/trends with no matching press coverage
249
+ """
250
+ # Find article URLs that were matched by at least one correlation
251
+ matched_article_urls: set[str] = set()
252
+ for corr in correlations:
253
+ matched_article_urls.update(corr.get("matched_articles", []))
254
+
255
+ # Unmatched articles → uncovered tech trends
256
+ unmatched_articles = [a for a in articles if a.get("url") not in matched_article_urls]
257
+
258
+ # Group unmatched articles by topic
259
+ topic_articles: dict[str, list[dict[str, Any]]] = {}
260
+ for article in unmatched_articles:
261
+ topic = _extract_article_topic(article)
262
+ topic_articles.setdefault(topic, []).append(article)
263
+
264
+ uncovered_tech_trends = [
265
+ {
266
+ "topic": topic,
267
+ "techcrunch_articles": [
268
+ {"title": a.get("title", ""), "url": a.get("url", "")}
269
+ for a in arts
270
+ ],
271
+ "signal": "No matching GitHub activity",
272
+ }
273
+ for topic, arts in sorted(topic_articles.items(), key=lambda x: -len(x[1]))
274
+ ]
275
+
276
+ # Find repos that had no correlation match
277
+ correlated_repo_names: set[str] = {c.get("repo", "") for c in correlations}
278
+ unmatched_repos = [
279
+ r for r in repos
280
+ if (r.get("full_name") or f"{r.get('owner')}/{r.get('name')}") not in correlated_repo_names
281
+ ]
282
+
283
+ # Group unmatched repos by topic
284
+ topic_repos: dict[str, list[dict[str, Any]]] = {}
285
+ for repo in unmatched_repos:
286
+ topic = _extract_repo_topic(repo)
287
+ topic_repos.setdefault(topic, []).append(repo)
288
+
289
+ unpublicized_dev_activity = [
290
+ {
291
+ "topic": topic,
292
+ "github_repos": [
293
+ {
294
+ "full_name": r.get("full_name") or f"{r.get('owner')}/{r.get('name')}",
295
+ "stars": r.get("stars", 0),
296
+ "stars_gained": r.get("stars_gained"),
297
+ }
298
+ for r in reps
299
+ ],
300
+ "signal": "No TechCrunch coverage",
301
+ }
302
+ for topic, reps in sorted(topic_repos.items(), key=lambda x: -len(x[1]))
303
+ ]
304
+
305
+ return {
306
+ "uncovered_tech_trends": uncovered_tech_trends,
307
+ "unpublicized_dev_activity": unpublicized_dev_activity,
308
+ }
309
+
310
+
311
def correlate_all(repos: list[dict[str, Any]], articles: list[dict[str, Any]], week: str) -> dict[str, Any]:
312
"""Run correlation engine across all repos and articles."""
313
correlations: list[dict[str, Any]] = []
@@ -230,14 +326,20 @@ def correlate_all(repos: list[dict[str, Any]], articles: list[dict[str, Any]], w
326
327
articles_matched = len({url for c in correlations for url in c["matched_articles"]})
328
329
+ # Detect divergences
330
+ divergences = detect_divergences(repos, articles, correlations)
331
+
332
return {
333
"week": week,
334
"correlations": correlations,
335
+ "divergences": divergences,
336
"uncorrelated_repos": uncorrelated,
337
"metadata": {
338
"repos_analyzed": len(repos),
339
"correlations_found": len(correlations),
340
"articles_matched": articles_matched,
341
+ "uncovered_tech_trends": len(divergences["uncovered_tech_trends"]),
342
+ "unpublicized_dev_activity": len(divergences["unpublicized_dev_activity"]),
343
},
344
}
345
scripts/render_press_context.py
+58
@@ -70,6 +70,54 @@ def format_correlations_list(correlations: list[dict]) -> str:
70
return "\n".join(lines)
71
72
73
+def format_divergences(divergences: dict) -> str:
74
+ """Format divergences section into markdown."""
75
+ if not divergences:
76
+ return ""
77
+
78
+ uncovered = divergences.get("uncovered_tech_trends", [])
79
+ unpublicized = divergences.get("unpublicized_dev_activity", [])
80
+
81
+ if not uncovered and not unpublicized:
82
+ return ""
83
+
84
+ lines = ["\n### Divergence Analysis\n"]
85
+
86
+ if uncovered:
87
+ lines.append("#### 🔍 Tech Trends Without Dev Activity")
88
+ lines.append("Topics heavily covered by TechCrunch with no matching GitHub repos:\n")
89
+ for item in uncovered:
90
+ topic = item.get("topic", "unknown")
91
+ articles = item.get("techcrunch_articles", [])
92
+ article_refs = ", ".join(
93
+ f"[{a.get('title', 'article')}]({a.get('url', '')})"
94
+ for a in articles[:3]
95
+ )
96
+ lines.append(f"- **{topic}**: {article_refs}")
97
+ lines.append("")
98
+
99
+ if unpublicized:
100
+ lines.append("#### 🚀 Dev Activity Without Press Coverage")
101
+ lines.append("GitHub repos/trends with no matching TechCrunch coverage:\n")
102
+ for item in unpublicized:
103
+ topic = item.get("topic", "unknown")
104
+ repos = item.get("github_repos", [])
105
+ repo_refs = ", ".join(
106
+ f"{r.get('full_name', '?')} (⭐{r.get('stars', 0)})"
107
+ for r in repos[:3]
108
+ )
109
+ lines.append(f"- **{topic}**: {repo_refs}")
110
+ lines.append("")
111
+
112
+ lines.append("#### Divergence Instructions")
113
+ lines.append("Use divergences to identify:")
114
+ lines.append("- 🔮 Where industry is moving but devs haven't caught up")
115
+ lines.append("- 💡 Where devs are innovating ahead of media attention")
116
+ lines.append("- 📊 Opportunity gaps between narrative and reality")
117
+
118
+ return "\n".join(lines)
119
+
120
+
121
def render_press_context(
122
techcrunch_data: dict | None, correlation_data: dict | None, week: str
123
) -> str:
@@ -108,6 +156,11 @@ def render_press_context(
156
article_count = len(articles)
157
correlation_count = len(correlations)
158
159
+ # Extract divergences
160
+ divergences = {}
161
+ if correlation_data:
162
+ divergences = correlation_data.get("divergences", {})
163
+
164
# Render template
165
rendered = template.replace("{date}", week)
166
rendered = rendered.replace("{article_count}", str(article_count))
@@ -115,6 +168,11 @@ def render_press_context(
168
rendered = rendered.replace("{correlation_count}", str(correlation_count))
169
rendered = rendered.replace("{correlations_list}", format_correlations_list(correlations))
170
171
+ # Append divergences section
172
+ divergence_section = format_divergences(divergences)
173
+ if divergence_section:
174
+ rendered += "\n" + divergence_section
175
+
176
return rendered
177
178