@cryptotaxi247 / SquadScope / commits / 5e51498

feat: TechCrunch RSS crawler with entity extraction (#74, #75) (#107)

- Add feedparser dependency to requirements.txt - Create scripts/techcrunch_crawler.py with TechCrunchSource class - Implement RSS feed fetching with retry, date filtering - Extract GitHub URLs via regex from article content - Extract entities (company/project names) from titles - Compute relevance scores based on tech keyword density - Output structured JSON with metadata to data/raw/{topic}/ - Add comprehensive test suite (23 tests, all mocked) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Juan Manuel Servera committed May 19, 2026 at 16:12 UTC 5e5149899854b1bfb6eb083ffbb94a6816dbe02f
3 files changed +580
requirements.txt
+1
@@ -1,3 +1,4 @@
1 # Core dependencies
2 pydantic>=2.0,<3.0
3 pyyaml>=6.0,<7.0
4 +feedparser>=6.0,<7.0
scripts/techcrunch_crawler.py new
+292
@@ -0,0 +1,292 @@
1 +#!/usr/bin/env python3
2 +"""TechCrunch RSS crawler with entity extraction for SquadScope.
3 +
4 +Fetches articles from TechCrunch RSS feed, extracts structured metadata,
5 +GitHub URLs, and entities (company/project names).
6 +
7 +Usage:
8 + python scripts/techcrunch_crawler.py [--topic ai-ml] \
9 + [--output data/raw/ai-ml/2026-W21-techcrunch.json] [--since 2026-05-11]
10 +"""
11 +
12 +from __future__ import annotations
13 +
14 +import argparse
15 +import json
16 +import re
17 +import sys
18 +import time
19 +from datetime import UTC, datetime, timedelta
20 +from pathlib import Path
21 +from typing import Any
22 +
23 +import feedparser
24 +
25 +from scripts.topic_paths import raw_dir
26 +
27 +FEED_URL = "https://techcrunch.com/feed/"
28 +
29 +GITHUB_URL_RE = re.compile(
30 + r"https?://github\.com/[a-zA-Z0-9_.\-]+/[a-zA-Z0-9_.\-]+"
31 +)
32 +
33 +TECH_KEYWORDS = {
34 + "ai", "ml", "machine learning", "deep learning", "open-source",
35 + "open source", "github", "developer", "api", "framework", "sdk",
36 + "llm", "gpt", "model", "neural", "transformer", "cloud", "devops",
37 + "kubernetes", "docker", "rust", "python", "javascript", "typescript",
38 + "golang", "database", "vector", "embedding", "agent", "rag",
39 + "fine-tuning", "inference", "startup", "oss",
40 +}
41 +
42 +# Common lowercase words that should not be treated as entities
43 +STOP_WORDS = {
44 + "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
45 + "of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
46 + "has", "have", "had", "do", "does", "did", "will", "would", "could",
47 + "should", "may", "might", "can", "this", "that", "these", "those",
48 + "it", "its", "new", "how", "why", "what", "when", "where", "who",
49 + "all", "just", "more", "most", "some", "any", "no", "not", "than",
50 + "too", "very", "also", "about", "up", "out", "into", "over", "after",
51 + "before", "between", "under", "again", "here", "there", "now", "then",
52 + "once", "well", "back", "still", "even", "big", "first", "last",
53 + "next", "says", "said", "gets", "got", "makes", "made", "takes",
54 + "took", "goes", "went", "comes", "came", "wants", "launches",
55 + "raises", "builds", "looks", "like", "use", "using", "used",
56 +}
57 +
58 +
59 +def iso_timestamp(value: datetime) -> str:
60 + """Format datetime as ISO 8601 UTC string."""
61 + return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
62 +
63 +
64 +def week_slug(value: datetime) -> str:
65 + """Return ISO week string like '2026-W21'."""
66 + year, week, _ = value.isocalendar()
67 + return f"{year}-W{week:02d}"
68 +
69 +
70 +def extract_github_urls(text: str) -> list[str]:
71 + """Extract unique GitHub repository URLs from text."""
72 + if not text:
73 + return []
74 + urls = GITHUB_URL_RE.findall(text)
75 + # Deduplicate while preserving order
76 + seen: set[str] = set()
77 + result: list[str] = []
78 + for url in urls:
79 + # Strip trailing periods/commas that may be captured
80 + url = url.rstrip(".,;)")
81 + if url not in seen:
82 + seen.add(url)
83 + result.append(url)
84 + return result
85 +
86 +
87 +def extract_entities(title: str) -> list[str]:
88 + """Extract likely entity names (companies, projects) from a title.
89 +
90 + Heuristic: capitalized words that aren't common English words.
91 + """
92 + if not title:
93 + return []
94 + words = re.findall(r"\b[A-Z][a-zA-Z0-9]*(?:\.[a-zA-Z]+)*\b", title)
95 + entities: list[str] = []
96 + seen: set[str] = set()
97 + for word in words:
98 + lower = word.lower()
99 + if lower in STOP_WORDS:
100 + continue
101 + if len(word) < 2:
102 + continue
103 + if word not in seen:
104 + seen.add(word)
105 + entities.append(word)
106 + return entities
107 +
108 +
109 +def compute_relevance_score(article: dict[str, Any]) -> float:
110 + """Compute a 0-1 relevance score based on tech/OSS keyword density."""
111 + text = " ".join([
112 + article.get("title", ""),
113 + article.get("summary", ""),
114 + " ".join(article.get("categories", [])),
115 + ]).lower()
116 +
117 + if not text.strip():
118 + return 0.0
119 +
120 + matches = sum(1 for kw in TECH_KEYWORDS if kw in text)
121 + # Normalize: cap at 1.0, scale so 5+ keywords = 1.0
122 + score = min(matches / 5.0, 1.0)
123 + # Boost if GitHub links found
124 + if article.get("github_links"):
125 + score = min(score + 0.2, 1.0)
126 + return round(score, 2)
127 +
128 +
129 +def parse_published_date(entry: Any) -> datetime | None:
130 + """Parse the published date from a feedparser entry."""
131 + published_parsed = getattr(entry, "published_parsed", None)
132 + if published_parsed:
133 + return datetime(*published_parsed[:6], tzinfo=UTC)
134 + # Fallback: try updated_parsed
135 + updated_parsed = getattr(entry, "updated_parsed", None)
136 + if updated_parsed:
137 + return datetime(*updated_parsed[:6], tzinfo=UTC)
138 + return None
139 +
140 +
141 +def fetch_feed(url: str = FEED_URL, retries: int = 1) -> Any:
142 + """Fetch and parse RSS feed with retry on failure."""
143 + for attempt in range(retries + 1):
144 + feed = feedparser.parse(url)
145 + if feed.bozo and not feed.entries:
146 + if attempt < retries:
147 + time.sleep(2)
148 + continue
149 + # Return partial result even on failure
150 + return feed
151 + return feed
152 + return feed # pragma: no cover
153 +
154 +
155 +class TechCrunchSource:
156 + """TechCrunch RSS data source following the DataSource protocol."""
157 +
158 + def get_name(self) -> str:
159 + return "techcrunch"
160 +
161 + def get_rate_limits(self) -> dict:
162 + return {"requests_per_minute": 10}
163 +
164 + def crawl(
165 + self,
166 + since: datetime,
167 + until: datetime,
168 + feed_url: str = FEED_URL,
169 + ) -> list[dict[str, Any]]:
170 + """Crawl TechCrunch RSS feed and return structured articles."""
171 + feed = fetch_feed(feed_url)
172 + articles: list[dict[str, Any]] = []
173 +
174 + for entry in feed.entries:
175 + pub_date = parse_published_date(entry)
176 + if pub_date is None:
177 + continue
178 + if pub_date < since or pub_date >= until:
179 + continue
180 +
181 + # Get content for GitHub URL extraction
182 + content_text = ""
183 + if hasattr(entry, "content") and entry.content:
184 + content_text = entry.content[0].get("value", "")
185 + elif hasattr(entry, "summary"):
186 + content_text = entry.summary or ""
187 +
188 + categories = [
189 + tag.term for tag in getattr(entry, "tags", [])
190 + if hasattr(tag, "term")
191 + ]
192 +
193 + summary = getattr(entry, "summary", "") or ""
194 + # Strip HTML tags from summary
195 + summary = re.sub(r"<[^>]+>", "", summary).strip()
196 + if len(summary) > 500:
197 + summary = summary[:497] + "..."
198 +
199 + article: dict[str, Any] = {
200 + "title": getattr(entry, "title", ""),
201 + "url": getattr(entry, "link", ""),
202 + "published_at": iso_timestamp(pub_date),
203 + "categories": categories,
204 + "summary": summary,
205 + "github_links": extract_github_urls(content_text),
206 + "entities": extract_entities(getattr(entry, "title", "")),
207 + }
208 + article["relevance_score"] = compute_relevance_score(article)
209 + articles.append(article)
210 +
211 + return articles
212 +
213 +
214 +def build_output(
215 + articles: list[dict[str, Any]],
216 + crawled_at: datetime,
217 +) -> dict[str, Any]:
218 + """Build the final output structure with metadata."""
219 + relevant = [a for a in articles if a["relevance_score"] >= 0.4]
220 + all_github_links = set()
221 + for a in articles:
222 + all_github_links.update(a.get("github_links", []))
223 +
224 + return {
225 + "week": week_slug(crawled_at),
226 + "source": "techcrunch",
227 + "crawled_at": iso_timestamp(crawled_at),
228 + "articles": articles,
229 + "metadata": {
230 + "total_articles": len(articles),
231 + "relevant_articles": len(relevant),
232 + "github_links_found": len(all_github_links),
233 + },
234 + }
235 +
236 +
237 +def main(argv: list[str] | None = None) -> int:
238 + parser = argparse.ArgumentParser(
239 + description="Crawl TechCrunch RSS feed for SquadScope"
240 + )
241 + parser.add_argument(
242 + "--topic", default="general",
243 + help="Topic ID for output path (default: general)",
244 + )
245 + parser.add_argument(
246 + "--output", default=None,
247 + help="Override output file path",
248 + )
249 + parser.add_argument(
250 + "--since", default=None,
251 + help="Start date filter (YYYY-MM-DD, default: 7 days ago)",
252 + )
253 + parser.add_argument(
254 + "--until", default=None,
255 + help="End date filter (YYYY-MM-DD, default: now)",
256 + )
257 + args = parser.parse_args(argv)
258 +
259 + now = datetime.now(UTC)
260 + since = (
261 + datetime.strptime(args.since, "%Y-%m-%d").replace(tzinfo=UTC)
262 + if args.since
263 + else now - timedelta(days=7)
264 + )
265 + until = (
266 + datetime.strptime(args.until, "%Y-%m-%d").replace(tzinfo=UTC)
267 + if args.until
268 + else now
269 + )
270 +
271 + source = TechCrunchSource()
272 + articles = source.crawl(since=since, until=until)
273 + output = build_output(articles, crawled_at=now)
274 +
275 + if args.output:
276 + out_path = Path(args.output)
277 + else:
278 + out_dir = raw_dir(args.topic)
279 + out_dir.mkdir(parents=True, exist_ok=True)
280 + out_path = out_dir / f"{week_slug(now)}-techcrunch.json"
281 +
282 + out_path.parent.mkdir(parents=True, exist_ok=True)
283 + with open(out_path, "w", encoding="utf-8") as f:
284 + json.dump(output, f, indent=2, ensure_ascii=False)
285 +
286 + print(f"Crawled {output['metadata']['total_articles']} articles "
287 + f"({output['metadata']['relevant_articles']} relevant) → {out_path}")
288 + return 0
289 +
290 +
291 +if __name__ == "__main__":
292 + sys.exit(main())
tests/test_techcrunch_crawler.py new
+287
@@ -0,0 +1,287 @@
1 +"""Tests for TechCrunch RSS crawler — no live feed fetching."""
2 +
3 +from __future__ import annotations
4 +
5 +from datetime import UTC, datetime, timedelta
6 +from types import SimpleNamespace
7 +from unittest.mock import patch
8 +
9 +import pytest
10 +
11 +from scripts.techcrunch_crawler import (
12 + TechCrunchSource,
13 + build_output,
14 + compute_relevance_score,
15 + extract_entities,
16 + extract_github_urls,
17 + iso_timestamp,
18 + parse_published_date,
19 + week_slug,
20 +)
21 +
22 +
23 +# --- Fixtures ---
24 +
25 +def _make_entry(
26 + title="Test Article",
27 + link="https://techcrunch.com/2026/05/15/test/",
28 + published_parsed=(2026, 5, 15, 10, 0, 0, 3, 135, 0),
29 + summary="A test summary about AI and open-source projects.",
30 + content_html="<p>Check out <a href='https://github.com/org/repo'>this repo</a></p>",
31 + tags=None,
32 +):
33 + entry = SimpleNamespace(
34 + title=title,
35 + link=link,
36 + published_parsed=published_parsed,
37 + summary=summary,
38 + content=[{"value": content_html}],
39 + tags=tags or [SimpleNamespace(term="AI"), SimpleNamespace(term="Startups")],
40 + )
41 + return entry
42 +
43 +
44 +def _make_feed(entries=None, bozo=False):
45 + return SimpleNamespace(entries=entries or [], bozo=bozo)
46 +
47 +
48 +# --- Unit tests: utility functions ---
49 +
50 +class TestIsoTimestamp:
51 + def test_basic(self):
52 + dt = datetime(2026, 5, 15, 10, 0, 0, tzinfo=UTC)
53 + assert iso_timestamp(dt) == "2026-05-15T10:00:00Z"
54 +
55 +
56 +class TestWeekSlug:
57 + def test_basic(self):
58 + dt = datetime(2026, 5, 15, tzinfo=UTC)
59 + result = week_slug(dt)
60 + assert result.startswith("2026-W")
61 + assert len(result) == 8
62 +
63 +
64 +# --- Unit tests: extraction ---
65 +
66 +class TestExtractGithubUrls:
67 + def test_finds_urls(self):
68 + text = "Check https://github.com/langchain-ai/langchain and https://github.com/openai/openai-python"
69 + result = extract_github_urls(text)
70 + assert "https://github.com/langchain-ai/langchain" in result
71 + assert "https://github.com/openai/openai-python" in result
72 +
73 + def test_deduplicates(self):
74 + text = "https://github.com/org/repo https://github.com/org/repo"
75 + result = extract_github_urls(text)
76 + assert len(result) == 1
77 +
78 + def test_empty_input(self):
79 + assert extract_github_urls("") == []
80 + assert extract_github_urls(None) == []
81 +
82 + def test_no_matches(self):
83 + assert extract_github_urls("No GitHub links here") == []
84 +
85 + def test_strips_trailing_punctuation(self):
86 + text = "See https://github.com/org/repo."
87 + result = extract_github_urls(text)
88 + assert result == ["https://github.com/org/repo"]
89 +
90 +
91 +class TestExtractEntities:
92 + def test_finds_proper_nouns(self):
93 + title = "OpenAI Launches New GPT Model for Developers"
94 + entities = extract_entities(title)
95 + assert "OpenAI" in entities
96 + assert "GPT" in entities
97 + assert "Model" in entities
98 + assert "Developers" in entities
99 +
100 + def test_skips_stop_words(self):
101 + title = "The New AI Framework"
102 + entities = extract_entities(title)
103 + # "The" and "New" are stop words
104 + assert "The" not in entities
105 + assert "New" not in entities
106 + assert "AI" in entities
107 + assert "Framework" in entities
108 +
109 + def test_empty(self):
110 + assert extract_entities("") == []
111 + assert extract_entities(None) == []
112 +
113 +
114 +class TestComputeRelevanceScore:
115 + def test_high_relevance(self):
116 + article = {
117 + "title": "AI startup launches open-source LLM framework",
118 + "summary": "A new developer API using machine learning",
119 + "categories": ["AI", "Open Source"],
120 + "github_links": ["https://github.com/org/repo"],
121 + }
122 + score = compute_relevance_score(article)
123 + assert score >= 0.8
124 +
125 + def test_low_relevance(self):
126 + article = {
127 + "title": "Company raises funding round",
128 + "summary": "The company announced a new funding round today.",
129 + "categories": ["Funding"],
130 + "github_links": [],
131 + }
132 + score = compute_relevance_score(article)
133 + assert score < 0.4
134 +
135 + def test_github_boost(self):
136 + article_no_gh = {
137 + "title": "AI tool released",
138 + "summary": "",
139 + "categories": [],
140 + "github_links": [],
141 + }
142 + article_with_gh = {
143 + **article_no_gh,
144 + "github_links": ["https://github.com/org/repo"],
145 + }
146 + score_no = compute_relevance_score(article_no_gh)
147 + score_with = compute_relevance_score(article_with_gh)
148 + assert score_with > score_no
149 +
150 +
151 +class TestParsePublishedDate:
152 + def test_published_parsed(self):
153 + entry = SimpleNamespace(published_parsed=(2026, 5, 15, 10, 0, 0, 3, 135, 0))
154 + result = parse_published_date(entry)
155 + assert result == datetime(2026, 5, 15, 10, 0, 0, tzinfo=UTC)
156 +
157 + def test_fallback_updated(self):
158 + entry = SimpleNamespace(
159 + published_parsed=None,
160 + updated_parsed=(2026, 5, 14, 8, 0, 0, 2, 134, 0),
161 + )
162 + result = parse_published_date(entry)
163 + assert result == datetime(2026, 5, 14, 8, 0, 0, tzinfo=UTC)
164 +
165 + def test_none_when_missing(self):
166 + entry = SimpleNamespace(published_parsed=None, updated_parsed=None)
167 + assert parse_published_date(entry) is None
168 +
169 +
170 +# --- Integration tests: TechCrunchSource.crawl ---
171 +
172 +class TestTechCrunchSourceCrawl:
173 + def test_crawl_filters_by_date(self):
174 + entry_in_range = _make_entry(
175 + published_parsed=(2026, 5, 15, 10, 0, 0, 3, 135, 0)
176 + )
177 + entry_out_of_range = _make_entry(
178 + title="Old Article",
179 + published_parsed=(2026, 4, 1, 10, 0, 0, 1, 91, 0),
180 + )
181 + feed = _make_feed(entries=[entry_in_range, entry_out_of_range])
182 +
183 + with patch("scripts.techcrunch_crawler.fetch_feed", return_value=feed):
184 + source = TechCrunchSource()
185 + articles = source.crawl(
186 + since=datetime(2026, 5, 10, tzinfo=UTC),
187 + until=datetime(2026, 5, 20, tzinfo=UTC),
188 + )
189 +
190 + assert len(articles) == 1
191 + assert articles[0]["title"] == "Test Article"
192 +
193 + def test_crawl_extracts_github_links(self):
194 + entry = _make_entry(
195 + content_html="<p>See https://github.com/pytorch/pytorch for details</p>"
196 + )
197 + feed = _make_feed(entries=[entry])
198 +
199 + with patch("scripts.techcrunch_crawler.fetch_feed", return_value=feed):
200 + source = TechCrunchSource()
201 + articles = source.crawl(
202 + since=datetime(2026, 5, 10, tzinfo=UTC),
203 + until=datetime(2026, 5, 20, tzinfo=UTC),
204 + )
205 +
206 + assert "https://github.com/pytorch/pytorch" in articles[0]["github_links"]
207 +
208 + def test_crawl_extracts_entities(self):
209 + entry = _make_entry(title="OpenAI and LangChain Release New Tools")
210 + feed = _make_feed(entries=[entry])
211 +
212 + with patch("scripts.techcrunch_crawler.fetch_feed", return_value=feed):
213 + source = TechCrunchSource()
214 + articles = source.crawl(
215 + since=datetime(2026, 5, 10, tzinfo=UTC),
216 + until=datetime(2026, 5, 20, tzinfo=UTC),
217 + )
218 +
219 + entities = articles[0]["entities"]
220 + assert "OpenAI" in entities
221 + assert "LangChain" in entities
222 +
223 + def test_crawl_strips_html_from_summary(self):
224 + entry = _make_entry(summary="<p>Hello <b>world</b></p>")
225 + feed = _make_feed(entries=[entry])
226 +
227 + with patch("scripts.techcrunch_crawler.fetch_feed", return_value=feed):
228 + source = TechCrunchSource()
229 + articles = source.crawl(
230 + since=datetime(2026, 5, 10, tzinfo=UTC),
231 + until=datetime(2026, 5, 20, tzinfo=UTC),
232 + )
233 +
234 + assert "<" not in articles[0]["summary"]
235 + assert "Hello world" == articles[0]["summary"]
236 +
237 +
238 +# --- Output structure tests ---
239 +
240 +class TestBuildOutput:
241 + def test_structure(self):
242 + articles = [
243 + {
244 + "title": "Test",
245 + "url": "https://techcrunch.com/test",
246 + "published_at": "2026-05-15T10:00:00Z",
247 + "categories": ["AI"],
248 + "summary": "Test",
249 + "github_links": ["https://github.com/org/repo"],
250 + "entities": ["OpenAI"],
251 + "relevance_score": 0.8,
252 + },
253 + {
254 + "title": "Low relevance",
255 + "url": "https://techcrunch.com/low",
256 + "published_at": "2026-05-15T11:00:00Z",
257 + "categories": [],
258 + "summary": "Funding",
259 + "github_links": [],
260 + "entities": [],
261 + "relevance_score": 0.2,
262 + },
263 + ]
264 + now = datetime(2026, 5, 19, 10, 0, 0, tzinfo=UTC)
265 + output = build_output(articles, crawled_at=now)
266 +
267 + assert output["source"] == "techcrunch"
268 + assert output["week"] == week_slug(now)
269 + assert output["crawled_at"] == "2026-05-19T10:00:00Z"
270 + assert output["metadata"]["total_articles"] == 2
271 + assert output["metadata"]["relevant_articles"] == 1
272 + assert output["metadata"]["github_links_found"] == 1
273 + assert len(output["articles"]) == 2
274 +
275 +
276 +# --- DataSource protocol tests ---
277 +
278 +class TestDataSourceProtocol:
279 + def test_get_name(self):
280 + source = TechCrunchSource()
281 + assert source.get_name() == "techcrunch"
282 +
283 + def test_get_rate_limits(self):
284 + source = TechCrunchSource()
285 + limits = source.get_rate_limits()
286 + assert "requests_per_minute" in limits
287 + assert limits["requests_per_minute"] == 10