feat: wire TechCrunch RSS into CI pipeline and add exponential backoff (#126)
- Add TechCrunch RSS crawl step to the crawl job (after GitHub crawl) - Add correlation and press context steps to the analyze job - Append press context to the AI analysis prompt - Add exponential backoff retry (max 3 retries, 2^n + jitter) to GitHub Models API calls in analyze_fallback.py - Retry on HTTP 429, 500, 502, 503, 504 with Retry-After support - Install Python deps in both crawl and analyze jobs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Juan Manuel Servera committed
May 19, 2026 at 19:29 UTC
f05297433e33aa3ccf4dba4ef28d0e996ebf87b4
2 files changed
+108
-20
.github/workflows/crawl-and-publish.yml
+44
@@ -93,6 +93,17 @@ jobs:
93
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
94
run: python scripts/crawl.py
95
96
+ - name: Install Python dependencies
97
+ run: pip install -r requirements.txt
98
+
99
+ - name: Crawl TechCrunch RSS
100
+ run: |
101
+ WEEK=$(date +%Y-W%V)
102
+ SINCE=$(date -d '7 days ago' +%Y-%m-%d)
103
+ python scripts/techcrunch_crawler.py \
104
+ --output "data/raw/${WEEK}-techcrunch.json" \
105
+ --since "$SINCE"
106
+
107
- name: Upload raw crawl artifact
108
if: always()
109
uses: actions/upload-artifact@v4
@@ -219,6 +230,32 @@ jobs:
230
echo "current_datetime=$CURRENT_DATETIME"
231
} >> "$GITHUB_OUTPUT"
232
233
+ - name: Install Python dependencies
234
+ run: pip install -r requirements.txt
235
+
236
+ - name: Run correlation and press context
237
+ id: press-context
238
+ run: |
239
+ set -euo pipefail
240
+ WEEK="${{ steps.analysis-context.outputs.week }}"
241
+ WEEK_FILE="${{ steps.analysis-context.outputs.week_file }}"
242
+ TC_FILE="data/raw/${WEEK}-techcrunch.json"
243
+
244
+ # Run correlate if TechCrunch data exists
245
+ if [ -f "$TC_FILE" ]; then
246
+ mkdir -p data/analyzed
247
+ python scripts/correlate.py \
248
+ --raw "$WEEK_FILE" \
249
+ --techcrunch "$TC_FILE" \
250
+ --output "data/analyzed/${WEEK}-correlations.json"
251
+ fi
252
+
253
+ # Render press context (handles missing files gracefully)
254
+ PRESS_CONTEXT=$(python scripts/render_press_context.py --week "$WEEK")
255
+ PRESS_FILE="data/analyzed/${WEEK}-press-context.md"
256
+ printf '%s\n' "$PRESS_CONTEXT" > "$PRESS_FILE"
257
+ echo "press_file=$PRESS_FILE" >> "$GITHUB_OUTPUT"
258
+
259
- name: Pre-flight cost check
260
id: preflight-cost
261
run: |
@@ -251,10 +288,17 @@ jobs:
288
WEEK_FILE="${{ steps.analysis-context.outputs.week_file }}"
289
WEEK="${{ steps.analysis-context.outputs.week }}"
290
CURRENT_DATETIME="${{ steps.analysis-context.outputs.current_datetime }}"
291
+ PRESS_FILE="${{ steps.press-context.outputs.press_file }}"
292
mkdir -p data/metrics
293
PROMPT_FILE=$(mktemp)
294
python3 scripts/analyze_fallback.py --raw-json "$WEEK_FILE" --output "$OUTPUT_FILE" --current-datetime "$CURRENT_DATETIME" --print-prompt > "$PROMPT_FILE"
295
296
+ # Append press context to prompt if available
297
+ if [ -f "$PRESS_FILE" ] && [ -s "$PRESS_FILE" ]; then
298
+ printf '\n\n---\n## Press Context\n\n' >> "$PROMPT_FILE"
299
+ cat "$PRESS_FILE" >> "$PROMPT_FILE"
300
+ fi
301
+
302
if command -v copilot >/dev/null 2>&1 && copilot -p "$(cat "$PROMPT_FILE")" \
303
-s \
304
--no-ask-user \
scripts/analyze_fallback.py
+64
-20
@@ -4,6 +4,9 @@ from __future__ import annotations
4
import argparse
5
import json
6
import os
7
+import random
8
+import sys
9
+import time
10
from pathlib import Path
11
from typing import Any
12
from urllib import error, request
@@ -164,6 +167,11 @@ def extract_markdown(response_payload: dict[str, Any]) -> str:
167
raise ValueError("GitHub Models response did not contain markdown output.")
168
169
170
+RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
171
+MAX_RETRIES = 3
172
+BASE_DELAY = 2 # seconds
173
+
174
+
175
def call_github_models(prompt: str) -> str:
176
token = os.environ.get("GITHUB_TOKEN")
177
if not token:
@@ -178,27 +186,63 @@ def call_github_models(prompt: str) -> str:
186
"temperature": 0.3,
187
}
188
body = json.dumps(payload).encode("utf-8")
181
- req = request.Request(
182
- endpoint,
183
- data=body,
184
- headers={
185
- "Authorization": f"Bearer {token}",
186
- "Content-Type": "application/json",
187
- "Accept": "application/json",
188
- },
189
- method="POST",
190
- )
189
192
- try:
193
- with request.urlopen(req, timeout=timeout) as response:
194
- response_payload = json.load(response)
195
- except error.HTTPError as exc: # pragma: no cover - exercised via message formatting
196
- detail = exc.read().decode("utf-8", errors="replace")
197
- raise RuntimeError(f"GitHub Models API request failed ({exc.code}): {detail}") from exc
198
- except error.URLError as exc: # pragma: no cover - network failures are environment-specific
199
- raise RuntimeError(f"GitHub Models API request failed: {exc.reason}") from exc
200
-
201
- return extract_markdown(response_payload)
190
+ last_exc: Exception | None = None
191
+ for attempt in range(MAX_RETRIES + 1):
192
+ req = request.Request(
193
+ endpoint,
194
+ data=body,
195
+ headers={
196
+ "Authorization": f"Bearer {token}",
197
+ "Content-Type": "application/json",
198
+ "Accept": "application/json",
199
+ },
200
+ method="POST",
201
+ )
202
+ try:
203
+ with request.urlopen(req, timeout=timeout) as response:
204
+ response_payload = json.load(response)
205
+ return extract_markdown(response_payload)
206
+ except error.HTTPError as exc:
207
+ if exc.code not in RETRYABLE_STATUS_CODES or attempt == MAX_RETRIES:
208
+ detail = exc.read().decode("utf-8", errors="replace")
209
+ raise RuntimeError(
210
+ f"GitHub Models API request failed ({exc.code}): {detail}"
211
+ ) from exc
212
+ # Determine delay: respect Retry-After header on 429
213
+ retry_after = exc.headers.get("Retry-After") if exc.code == 429 else None
214
+ if retry_after is not None:
215
+ try:
216
+ delay = float(retry_after)
217
+ except ValueError:
218
+ delay = BASE_DELAY ** (attempt + 1)
219
+ else:
220
+ delay = BASE_DELAY ** (attempt + 1)
221
+ jitter = random.uniform(0, 1) # noqa: S311
222
+ total_delay = delay + jitter
223
+ print(
224
+ f"[retry] GitHub Models API returned {exc.code}, "
225
+ f"retrying in {total_delay:.1f}s (attempt {attempt + 1}/{MAX_RETRIES})",
226
+ file=sys.stderr,
227
+ )
228
+ last_exc = exc
229
+ time.sleep(total_delay)
230
+ except error.URLError as exc:
231
+ if attempt == MAX_RETRIES:
232
+ raise RuntimeError(
233
+ f"GitHub Models API request failed: {exc.reason}"
234
+ ) from exc
235
+ delay = BASE_DELAY ** (attempt + 1) + random.uniform(0, 1) # noqa: S311
236
+ print(
237
+ f"[retry] GitHub Models API network error: {exc.reason}, "
238
+ f"retrying in {delay:.1f}s (attempt {attempt + 1}/{MAX_RETRIES})",
239
+ file=sys.stderr,
240
+ )
241
+ last_exc = exc
242
+ time.sleep(delay)
243
+
244
+ # Should not be reached, but satisfy type checkers
245
+ raise RuntimeError("GitHub Models API request failed after retries") from last_exc
246
247
248
def generate_no_ai_summary(raw_json_path: Path, current_datetime: str) -> str: