fix: preserve external news review telemetry
Follow-up to #242 for issue #237 review fixes.
Juan Manuel Servera committed
Jun 5, 2026 at 19:34 UTC
f73714e246a9752e705f0b03818cee3e8f79ab83
6 files changed
+77
-1
.squad/agents/bender/history.md
+1
@@ -41,3 +41,4 @@
41
42
- Multi-source RSS remains in-process, but downstream reliability depends on a versioned canonical artifact: include crawl window, source config checksum, per-source statuses, partial failures, dedupe count, and deterministic checksum in `*-external-news.json`.
43
- Press correlation must retain bounded source-aware citations and label category/fuzzy-only matches as weak so mirrored coverage or broad topics do not inflate strong press claims.
44
+- PR #242 review follow-up: failed RSS fetch exceptions must stamp source-level attempt/timeout telemetry before re-raising, and scheduled external-news crawls should pass both `--since` and `--until` so canonical crawl windows are reproducible.
.squad/decisions/inbox/bender-pr-242-copilot-review-fixes.md
new
+6
@@ -0,0 +1,6 @@
1
+# Bender PR #242 Copilot Review Fixes
2
+
3
+- Keep category/project-name-only press matches weak even when temporally spiking or corroborated by multiple articles/sources.
4
+- Pass both `--since` and `--until` from the crawl workflow to preserve deterministic canonical `crawl_window` metadata.
5
+- Record bounded fetch attempts and timeout telemetry on `NewsFeedSource` even when `fetch_feed()` raises before returning a feed.
6
+- Keep press-context article lookup comments aligned with the actual URL-to-title mapping.
scripts/techcrunch_crawler.py
+6
-1
@@ -304,7 +304,12 @@ class NewsFeedSource:
304
"""Crawl an RSS feed and return structured articles."""
305
resolved_feed_url = feed_url or self.config.feed_url
306
validate_feed_url(resolved_feed_url)
307
- feed = fetch_feed(resolved_feed_url)
307
+ try:
308
+ feed = fetch_feed(resolved_feed_url)
309
+ except Exception:
310
+ self.last_attempts = DEFAULT_FETCH_RETRIES + 1
311
+ self.last_timeout_seconds = DEFAULT_FETCH_TIMEOUT_SECONDS
312
+ raise
313
self.last_attempts = int(getattr(feed, "squad_fetch_attempts", 1))
314
self.last_timeout_seconds = int(
315
getattr(feed, "squad_fetch_timeout_seconds", DEFAULT_FETCH_TIMEOUT_SECONDS)
tests/test_correlate.py
+32
@@ -11,6 +11,7 @@ from scripts.correlate import (
11
assess_hype_risk,
12
correlate_all,
13
correlate_repo,
14
+ correlation_strength,
15
dedupe_articles,
16
extract_week_from_filename,
17
fuzzy_name_score,
@@ -264,6 +265,37 @@ class TestCorrelateRepo:
265
assert result is not None
266
assert result["correlation_strength"] == "weak"
267
268
+ @pytest.mark.parametrize("match_type", ["category", "project_name"])
269
+ def test_weak_match_types_never_become_strong(self, match_type):
270
+ articles = [
271
+ _article(url="https://example.com/a", source="alpha"),
272
+ _article(url="https://example.com/b", source="beta"),
273
+ ]
274
+
275
+ assert correlation_strength(match_type, articles, temporal_spike=True) == "weak"
276
+
277
+ def test_corroborated_project_name_match_stays_weak(self):
278
+ repo = _repo(owner="acme", name="signal-kit", stars_gained=50)
279
+ articles = [
280
+ _article(
281
+ title="Signal Kit draws developer interest",
282
+ url="https://example.com/a",
283
+ entities=[],
284
+ source="alpha",
285
+ ),
286
+ _article(
287
+ title="Signal Kit keeps growing",
288
+ url="https://example.com/b",
289
+ entities=[],
290
+ source="beta",
291
+ ),
292
+ ]
293
+
294
+ result = correlate_repo(repo, articles)
295
+ assert result is not None
296
+ assert result["match_type"] == "project_name"
297
+ assert result["correlation_strength"] == "weak"
298
+
299
300
# ---------------------------------------------------------------------------
301
# Integration: correlate_all
tests/test_pipeline.py
+18
@@ -173,6 +173,24 @@ class WorkflowConfigTests(unittest.TestCase):
173
self.assertIn(".squad/run-counter.txt", run_script)
174
self.assertIn("git add data/raw/ data/snapshots/ .squad/run-counter.txt", run_script)
175
176
+ def test_external_news_workflow_passes_deterministic_until(self) -> None:
177
+ workflow_path = Path(".github/workflows/crawl-and-publish.yml")
178
+ workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8"))
179
+
180
+ crawl_job = workflow["jobs"]["crawl"]
181
+ external_news_step = next(
182
+ (
183
+ step for step in crawl_job["steps"]
184
+ if step.get("name") == "Crawl external news RSS feeds"
185
+ ),
186
+ None,
187
+ )
188
+
189
+ self.assertIsNotNone(external_news_step, "External news crawl step not found")
190
+ run_script = external_news_step["run"]
191
+ self.assertIn("UNTIL=$(date +%Y-%m-%d)", run_script)
192
+ self.assertIn('--until "$UNTIL"', run_script)
193
+
194
def test_crawl_workflow_defines_reskill_jobs(self) -> None:
195
workflow_path = Path(".github/workflows/crawl-and-publish.yml")
196
workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8"))
tests/test_techcrunch_crawler.py
+14
@@ -13,6 +13,7 @@ from scripts.techcrunch_crawler import (
13
DEFAULT_SOURCES_PATH,
14
DEFAULT_FETCH_TIMEOUT_SECONDS,
15
DEFAULT_FETCH_RETRIES,
16
+ NewsFeedSource,
17
NewsSourceConfig,
18
TechCrunchSource,
19
build_output,
@@ -536,3 +537,16 @@ class TestExternalNewsSources:
537
assert errors[0]["error_class"] == "TimeoutError"
538
assert statuses[0]["attempts"] == DEFAULT_FETCH_RETRIES + 1
539
assert statuses[0]["success"] is False
540
+
541
+ def test_failed_feed_fetch_records_attempts_on_source(self):
542
+ source = NewsFeedSource(NewsSourceConfig("alpha", "https://techcrunch.com/feed/"))
543
+
544
+ with patch("scripts.techcrunch_crawler.fetch_feed", side_effect=TimeoutError("boom")):
545
+ with pytest.raises(TimeoutError):
546
+ source.crawl(
547
+ since=datetime(2026, 5, 10, tzinfo=UTC),
548
+ until=datetime(2026, 5, 20, tzinfo=UTC),
549
+ )
550
+
551
+ assert source.last_attempts == DEFAULT_FETCH_RETRIES + 1
552
+ assert source.last_timeout_seconds == DEFAULT_FETCH_TIMEOUT_SECONDS