| 1 | name: Crawl and publish weekly data |
| 2 | |
| 3 | on: |
| 4 | schedule: |
| 5 | # Mondays at 00:07 as the low-effort default. On GitHub-hosted |
| 6 | # runners this schedule is best-effort and can start hours late; see |
| 7 | # docs/operator-guide.md#schedule-latency-and-mitigation-ladder and use |
| 8 | # workflow_dispatch from an external scheduler if punctuality matters. |
| 9 | - cron: '07 0 * * 1' |
| 10 | workflow_dispatch: |
| 11 | inputs: |
| 12 | run_mode: |
| 13 | description: 'Rerun mode: normal is guarded/non-destructive; dry-run/candidate-only never publish; restore requires rebuild_week; force-replace is explicit replacement.' |
| 14 | required: false |
| 15 | default: normal |
| 16 | type: choice |
| 17 | options: |
| 18 | - normal |
| 19 | - dry-run |
| 20 | - restore |
| 21 | - force-replace |
| 22 | - candidate-only |
| 23 | source_refresh_policy: |
| 24 | description: 'Source refresh policy for reruns. Default reuses eligible same-day source artifacts and refreshes missing/stale sources.' |
| 25 | required: false |
| 26 | default: reuse-same-day |
| 27 | type: choice |
| 28 | options: |
| 29 | - reuse-same-day |
| 30 | - refresh-missing-stale |
| 31 | - force-refresh |
| 32 | publish_release: |
| 33 | description: 'Create or update the weekly GitHub Release during a manual run. Not allowed for dry-run/candidate-only.' |
| 34 | required: false |
| 35 | default: false |
| 36 | type: boolean |
| 37 | rebuild_week: |
| 38 | description: 'Restore a specific past week (e.g. 2026-W21). Requires run_mode=restore and source_run_id; skips crawl and hydrates immutable raw evidence from publish.' |
| 39 | required: false |
| 40 | default: '' |
| 41 | type: string |
| 42 | source_run_id: |
| 43 | description: 'Required for restore: source workflow run ID selecting data/raw-store/<week>/<source_run_id>/ on publish.' |
| 44 | required: false |
| 45 | default: '' |
| 46 | type: string |
| 47 | force_reason: |
| 48 | description: 'Audited reason to force replacement while preserving all gates except candidate-vs-published score comparison.' |
| 49 | required: false |
| 50 | default: '' |
| 51 | type: string |
| 52 | analysis_path: |
| 53 | description: 'Analysis path. map-reduce-dry-run is allowed only with dry-run/candidate-only and never promotes content.' |
| 54 | required: false |
| 55 | default: single-pass |
| 56 | type: choice |
| 57 | options: |
| 58 | - single-pass |
| 59 | - map-reduce-dry-run |
| 60 | |
| 61 | permissions: |
| 62 | contents: read |
| 63 | |
| 64 | concurrency: |
| 65 | group: weekly-crawl |
| 66 | cancel-in-progress: false |
| 67 | |
| 68 | env: |
| 69 | PYTHONPATH: ${{ github.workspace }} |
| 70 | |
| 71 | jobs: |
| 72 | crawl: |
| 73 | runs-on: ubuntu-latest |
| 74 | permissions: |
| 75 | actions: read |
| 76 | contents: write |
| 77 | |
| 78 | steps: |
| 79 | - name: Check out repository # zizmor: ignore[artipacked] crawl job pushes to the publish branch; checkout token is reused by a later git push |
| 80 | uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 |
| 81 | with: |
| 82 | fetch-depth: 0 |
| 83 | ref: ${{ github.event.repository.default_branch }} |
| 84 | |
| 85 | - name: Validate rerun mode |
| 86 | id: rerun-mode |
| 87 | env: |
| 88 | RUN_MODE: ${{ inputs.run_mode || 'normal' }} |
| 89 | SOURCE_REFRESH_POLICY: ${{ inputs.source_refresh_policy || 'reuse-same-day' }} |
| 90 | REBUILD_WEEK: ${{ inputs.rebuild_week || '' }} |
| 91 | SOURCE_RUN_ID: ${{ inputs.source_run_id || '' }} |
| 92 | PUBLISH_RELEASE: ${{ inputs.publish_release || false }} |
| 93 | run: | |
| 94 | set -euo pipefail |
| 95 | ARGS=(--run-mode "$RUN_MODE" --source-refresh-policy "$SOURCE_REFRESH_POLICY" --rebuild-week "$REBUILD_WEEK" --source-run-id "$SOURCE_RUN_ID" --summary-json data/diagnostics/rerun-mode.json) |
| 96 | if [ "$PUBLISH_RELEASE" = "true" ]; then |
| 97 | ARGS+=(--publish-release) |
| 98 | fi |
| 99 | python3 scripts/rerun_modes.py "${ARGS[@]}" |
| 100 | echo "run_mode=$RUN_MODE" >> "$GITHUB_OUTPUT" |
| 101 | echo "source_refresh_policy=$SOURCE_REFRESH_POLICY" >> "$GITHUB_OUTPUT" |
| 102 | |
| 103 | - name: Find latest successful crawl cache |
| 104 | id: previous-cache-run |
| 105 | uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 |
| 106 | with: |
| 107 | script: | |
| 108 | const workflowId = 'crawl-and-publish.yml'; |
| 109 | const branch = context.payload.repository.default_branch; |
| 110 | const currentRunId = String(context.runId); |
| 111 | const perPage = 100; |
| 112 | let page = 1; |
| 113 | let previous = null; |
| 114 | |
| 115 | try { |
| 116 | while (!previous) { |
| 117 | const { data } = await github.rest.actions.listWorkflowRuns({ |
| 118 | owner: context.repo.owner, |
| 119 | repo: context.repo.repo, |
| 120 | workflow_id: workflowId, |
| 121 | branch, |
| 122 | status: 'completed', |
| 123 | per_page: perPage, |
| 124 | page, |
| 125 | }); |
| 126 | |
| 127 | previous = data.workflow_runs.find((run) => run.conclusion === 'success' && String(run.id) !== currentRunId) ?? null; |
| 128 | if (previous || data.workflow_runs.length < perPage) { |
| 129 | break; |
| 130 | } |
| 131 | |
| 132 | page += 1; |
| 133 | } |
| 134 | |
| 135 | core.setOutput('run_id', previous ? String(previous.id) : ''); |
| 136 | core.info(previous ? `Restoring crawl cache from run ${previous.id}.` : 'No previous successful crawl cache found.'); |
| 137 | } catch (error) { |
| 138 | core.warning(`Skipping cache restore lookup: ${error.message}`); |
| 139 | core.setOutput('run_id', ''); |
| 140 | } |
| 141 | |
| 142 | - name: Restore previous crawl cache |
| 143 | if: steps.previous-cache-run.outputs.run_id != '' |
| 144 | continue-on-error: true |
| 145 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 146 | with: |
| 147 | github-token: ${{ secrets.GITHUB_TOKEN }} |
| 148 | run-id: ${{ steps.previous-cache-run.outputs.run_id }} |
| 149 | name: crawl-cache |
| 150 | path: data/cache/ |
| 151 | |
| 152 | - name: Download previous raw artifacts for same-day reuse |
| 153 | if: ${{ steps.previous-cache-run.outputs.run_id != '' && !inputs.rebuild_week }} |
| 154 | continue-on-error: true |
| 155 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 156 | with: |
| 157 | github-token: ${{ secrets.GITHUB_TOKEN }} |
| 158 | run-id: ${{ steps.previous-cache-run.outputs.run_id }} |
| 159 | name: raw-data |
| 160 | path: .artifact-reuse/raw-data/ |
| 161 | |
| 162 | - name: Download previous snapshots for same-day reuse |
| 163 | if: ${{ steps.previous-cache-run.outputs.run_id != '' && !inputs.rebuild_week }} |
| 164 | continue-on-error: true |
| 165 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 166 | with: |
| 167 | github-token: ${{ secrets.GITHUB_TOKEN }} |
| 168 | run-id: ${{ steps.previous-cache-run.outputs.run_id }} |
| 169 | name: crawl-snapshots |
| 170 | path: .artifact-reuse/snapshots/ |
| 171 | |
| 172 | - name: Set up Python |
| 173 | uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 |
| 174 | with: |
| 175 | python-version: '3.12' |
| 176 | |
| 177 | - name: Restore immutable raw evidence |
| 178 | if: ${{ inputs.rebuild_week }} |
| 179 | env: |
| 180 | REBUILD_WEEK: ${{ inputs.rebuild_week }} |
| 181 | SOURCE_RUN_ID: ${{ inputs.source_run_id }} |
| 182 | run: | |
| 183 | set -euo pipefail |
| 184 | git fetch origin publish |
| 185 | RAW_STORE_DIR="data/raw-store/${REBUILD_WEEK}/${SOURCE_RUN_ID}" |
| 186 | git checkout origin/publish -- "$RAW_STORE_DIR" |
| 187 | python3 scripts/publish_safety.py restore-raw \ |
| 188 | --week "$REBUILD_WEEK" \ |
| 189 | --source-run-id "$SOURCE_RUN_ID" |
| 190 | |
| 191 | - name: Run crawler |
| 192 | if: ${{ !inputs.rebuild_week }} |
| 193 | env: |
| 194 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 195 | SOURCE_REFRESH_POLICY: ${{ steps.rerun-mode.outputs.source_refresh_policy }} |
| 196 | run: | |
| 197 | set -euo pipefail |
| 198 | WEEK=$(date -u +%Y-W%V) |
| 199 | CURRENT_DATETIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) |
| 200 | CODE_SHA=$(git hash-object scripts/crawl.py squadscope.topic.yml 2>/dev/null | sha256sum | cut -d' ' -f1) |
| 201 | python scripts/crawl.py \ |
| 202 | --reuse-artifact ".artifact-reuse/raw-data/${WEEK}.json" \ |
| 203 | --source-refresh-policy "$SOURCE_REFRESH_POLICY" \ |
| 204 | --run-started-at "$CURRENT_DATETIME" \ |
| 205 | --current-code-sha "$CODE_SHA" |
| 206 | |
| 207 | - name: Install Python dependencies |
| 208 | run: pip install -r requirements.txt |
| 209 | |
| 210 | - name: Crawl external news RSS feeds |
| 211 | if: ${{ !inputs.rebuild_week }} |
| 212 | env: |
| 213 | SOURCE_REFRESH_POLICY: ${{ steps.rerun-mode.outputs.source_refresh_policy }} |
| 214 | run: | |
| 215 | WEEK=$(date -u +%Y-W%V) |
| 216 | CURRENT_DATETIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) |
| 217 | SINCE=$(date -u -d '7 days ago' +%Y-%m-%d) |
| 218 | UNTIL=$(date -u +%Y-%m-%d) |
| 219 | CODE_SHA=$(git hash-object scripts/techcrunch_crawler.py config/external_news_sources.json 2>/dev/null | sha256sum | cut -d' ' -f1) |
| 220 | python scripts/techcrunch_crawler.py \ |
| 221 | --sources config/external_news_sources.json \ |
| 222 | --output "data/raw/${WEEK}-external-news.json" \ |
| 223 | --since "$SINCE" \ |
| 224 | --until "$UNTIL" \ |
| 225 | --reuse-artifact ".artifact-reuse/raw-data/${WEEK}-external-news.json" \ |
| 226 | --source-refresh-policy "$SOURCE_REFRESH_POLICY" \ |
| 227 | --run-started-at "$CURRENT_DATETIME" \ |
| 228 | --current-code-sha "$CODE_SHA" |
| 229 | python3 - <<'PY' "data/raw/${WEEK}-external-news.json" |
| 230 | import json |
| 231 | import sys |
| 232 | from pathlib import Path |
| 233 | |
| 234 | path = Path(sys.argv[1]) |
| 235 | payload = json.loads(path.read_text(encoding="utf-8")) |
| 236 | metadata = payload.get("metadata", {}) |
| 237 | print( |
| 238 | "::notice::External news artifact " |
| 239 | f"size_bytes={path.stat().st_size} " |
| 240 | f"articles={metadata.get('total_articles', 0)} " |
| 241 | f"relevant={metadata.get('relevant_articles', 0)} " |
| 242 | f"dedupe={metadata.get('dedupe_count', 0)} " |
| 243 | f"checksum={metadata.get('artifact_checksum', '')[:12]}" |
| 244 | ) |
| 245 | PY |
| 246 | |
| 247 | - name: Upload raw crawl artifact |
| 248 | id: raw-artifact |
| 249 | if: always() |
| 250 | uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| 251 | with: |
| 252 | name: raw-data |
| 253 | path: data/raw/ |
| 254 | if-no-files-found: warn |
| 255 | retention-days: 90 |
| 256 | |
| 257 | - name: Upload snapshot artifact |
| 258 | if: always() |
| 259 | uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| 260 | with: |
| 261 | name: crawl-snapshots |
| 262 | path: data/snapshots/ |
| 263 | if-no-files-found: warn |
| 264 | |
| 265 | - name: Upload cache artifact |
| 266 | if: always() |
| 267 | uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| 268 | with: |
| 269 | name: crawl-cache |
| 270 | path: data/cache/ |
| 271 | if-no-files-found: warn |
| 272 | |
| 273 | - name: Commit crawl data to data branch |
| 274 | if: ${{ !inputs.rebuild_week && inputs.run_mode != 'dry-run' && inputs.run_mode != 'candidate-only' }} |
| 275 | env: |
| 276 | DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} |
| 277 | DATA_BRANCH: publish |
| 278 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 279 | RAW_ARTIFACT_ID: ${{ steps.raw-artifact.outputs.artifact-id }} |
| 280 | SOURCE_HEAD_SHA: ${{ github.sha }} |
| 281 | run: | |
| 282 | set -euo pipefail |
| 283 | git config user.name "github-actions[bot]" |
| 284 | git config user.email "github-actions[bot]@users.noreply.github.com" |
| 285 | cp scripts/publish_safety.py publish-safety-tool.py |
| 286 | # Save crawl output before switching branches |
| 287 | cp -r data/raw crawl-raw-backup |
| 288 | cp -r data/snapshots crawl-snapshots-backup |
| 289 | # Fetch or create the unprotected data branch |
| 290 | if git fetch origin "$DATA_BRANCH" 2>/dev/null; then |
| 291 | EXPECTED_PUBLISH_SHA=$(git rev-parse "origin/$DATA_BRANCH") |
| 292 | git checkout -f -B "$DATA_BRANCH" "origin/$DATA_BRANCH" |
| 293 | else |
| 294 | EXPECTED_PUBLISH_SHA="" |
| 295 | git checkout -f -B "$DATA_BRANCH" "origin/$DEFAULT_BRANCH" |
| 296 | fi |
| 297 | # Restore crawl output on top of data branch |
| 298 | mkdir -p data/raw data/snapshots |
| 299 | cp -r crawl-raw-backup/* data/raw/ 2>/dev/null || true |
| 300 | cp -r crawl-snapshots-backup/* data/snapshots/ 2>/dev/null || true |
| 301 | rm -rf crawl-raw-backup crawl-snapshots-backup |
| 302 | WEEK=$(date -u +%Y-W%V) |
| 303 | RAW_PATH_ARGS=(--path "data/raw/${WEEK}.json") |
| 304 | for optional_raw in \ |
| 305 | "data/raw/${WEEK}-external-news.json" \ |
| 306 | "data/raw/${WEEK}-techcrunch.json" |
| 307 | do |
| 308 | [ -f "$optional_raw" ] && RAW_PATH_ARGS+=(--path "$optional_raw") |
| 309 | done |
| 310 | python3 publish-safety-tool.py store-raw \ |
| 311 | --week "$WEEK" \ |
| 312 | --source-run-id "$GITHUB_RUN_ID" \ |
| 313 | --source-artifact-id "$RAW_ARTIFACT_ID" \ |
| 314 | --source-artifact-name raw-data \ |
| 315 | --source-head-sha "$SOURCE_HEAD_SHA" \ |
| 316 | "${RAW_PATH_ARGS[@]}" |
| 317 | rm -f publish-safety-tool.py |
| 318 | git add data/raw/ data/snapshots/ data/raw-store/ |
| 319 | if git diff --cached --quiet; then |
| 320 | echo "No raw or snapshot changes to commit after staging." |
| 321 | exit 0 |
| 322 | fi |
| 323 | COUNTER=$(cat .squad/run-counter.txt 2>/dev/null || echo 0) |
| 324 | COUNTER=$((COUNTER + 1)) |
| 325 | printf '%s\n' "$COUNTER" > .squad/run-counter.txt |
| 326 | git add .squad/run-counter.txt |
| 327 | git diff --cached --quiet && exit 0 |
| 328 | git commit -m "data: weekly crawl $WEEK [run #${GITHUB_RUN_ID}]" |
| 329 | if [ -n "$EXPECTED_PUBLISH_SHA" ]; then |
| 330 | git push --force-with-lease="refs/heads/$DATA_BRANCH:$EXPECTED_PUBLISH_SHA" origin HEAD:"$DATA_BRANCH" |
| 331 | else |
| 332 | git push origin HEAD:"$DATA_BRANCH" |
| 333 | fi |
| 334 | |
| 335 | analyze: |
| 336 | needs: crawl |
| 337 | runs-on: ubuntu-latest |
| 338 | permissions: |
| 339 | actions: read |
| 340 | contents: write |
| 341 | issues: write |
| 342 | models: read |
| 343 | outputs: |
| 344 | week: ${{ steps.analysis-context.outputs.week }} |
| 345 | summary_file: ${{ steps.analysis-context.outputs.published_output_file }} |
| 346 | candidate_summary_file: ${{ steps.analysis-context.outputs.candidate_output_file }} |
| 347 | publish_manifest_file: ${{ steps.analysis-context.outputs.publish_manifest_file }} |
| 348 | current_datetime: ${{ steps.analysis-context.outputs.current_datetime }} |
| 349 | publish_base_sha: ${{ steps.publish-base.outputs.sha }} |
| 350 | publish_head_sha: ${{ steps.publish-base.outputs.sha }} |
| 351 | run_mode: ${{ steps.analysis-context.outputs.run_mode }} |
| 352 | |
| 353 | steps: |
| 354 | - name: Check out repository |
| 355 | uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 |
| 356 | with: |
| 357 | fetch-depth: 0 |
| 358 | ref: ${{ github.event.repository.default_branch }} |
| 359 | persist-credentials: false |
| 360 | |
| 361 | - name: Download raw crawl artifact |
| 362 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 363 | with: |
| 364 | name: raw-data |
| 365 | path: data/raw/ |
| 366 | |
| 367 | - name: Hydrate from publish (rebuild mode) |
| 368 | if: ${{ inputs.rebuild_week }} |
| 369 | env: |
| 370 | REBUILD_WEEK: ${{ inputs.rebuild_week }} |
| 371 | SOURCE_RUN_ID: ${{ inputs.source_run_id }} |
| 372 | run: | |
| 373 | set -euo pipefail |
| 374 | if ! [[ "$REBUILD_WEEK" =~ ^[0-9]{4}-W[0-9]{2}$ ]]; then |
| 375 | echo "::error::Invalid rebuild_week format. Expected YYYY-WNN (e.g. 2026-W21), got: $REBUILD_WEEK" |
| 376 | exit 1 |
| 377 | fi |
| 378 | git fetch origin publish |
| 379 | mkdir -p data/analyzed data/snapshots |
| 380 | git checkout origin/publish -- "data/raw-store/${REBUILD_WEEK}/${SOURCE_RUN_ID}" |
| 381 | python3 scripts/publish_safety.py restore-raw \ |
| 382 | --week "$REBUILD_WEEK" \ |
| 383 | --source-run-id "$SOURCE_RUN_ID" |
| 384 | git checkout origin/publish -- "data/snapshots/${REBUILD_WEEK}-stars.json" 2>/dev/null || true |
| 385 | # Also hydrate ALL prior analyzed files so rollups have correct historical context |
| 386 | git ls-tree -r --name-only origin/publish -- data/analyzed/ | while read -r f; do |
| 387 | git checkout origin/publish -- "$f" 2>/dev/null || true |
| 388 | done |
| 389 | |
| 390 | - name: Record publish branch base |
| 391 | id: publish-base |
| 392 | run: | |
| 393 | set -euo pipefail |
| 394 | if git fetch origin publish 2>/dev/null; then |
| 395 | echo "sha=$(git rev-parse origin/publish)" >> "$GITHUB_OUTPUT" |
| 396 | else |
| 397 | echo "sha=" >> "$GITHUB_OUTPUT" |
| 398 | fi |
| 399 | |
| 400 | - name: Set up Node |
| 401 | uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 |
| 402 | with: |
| 403 | node-version: '24' |
| 404 | |
| 405 | - name: Set up Python |
| 406 | uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 |
| 407 | with: |
| 408 | python-version: '3.12' |
| 409 | |
| 410 | - name: Prepare analysis context |
| 411 | id: analysis-context |
| 412 | env: |
| 413 | REBUILD_WEEK: ${{ inputs.rebuild_week || '' }} |
| 414 | RUN_MODE: ${{ inputs.run_mode || 'normal' }} |
| 415 | SOURCE_REFRESH_POLICY: ${{ inputs.source_refresh_policy || 'reuse-same-day' }} |
| 416 | run: | |
| 417 | mkdir -p data/analyzed data/candidates |
| 418 | CURRENT_DATETIME=$(date -u +%Y-%m-%dT%H:%M:%SZ) |
| 419 | readarray -t CONTEXT_LINES < <(python3 - <<'PY' "$CURRENT_DATETIME" |
| 420 | import json |
| 421 | import os |
| 422 | import sys |
| 423 | from datetime import UTC, datetime |
| 424 | from pathlib import Path |
| 425 | |
| 426 | current_datetime = sys.argv[1] |
| 427 | run_datetime = datetime.fromisoformat(current_datetime.replace("Z", "+00:00")).astimezone(UTC) |
| 428 | rebuild_week = os.environ.get("REBUILD_WEEK", "").strip() |
| 429 | if rebuild_week: |
| 430 | week = rebuild_week |
| 431 | else: |
| 432 | iso_year, iso_week, _ = run_datetime.isocalendar() |
| 433 | week = f"{iso_year}-W{iso_week:02d}" |
| 434 | week_file = Path("data/raw") / f"{week}.json" |
| 435 | if not week_file.exists(): |
| 436 | raise SystemExit(f"Missing raw payload for current run: {week_file}") |
| 437 | payload = json.loads(week_file.read_text(encoding="utf-8")) |
| 438 | if payload.get("week") != week: |
| 439 | raise SystemExit(f"Raw payload week mismatch: expected {week}, found {payload.get('week')!r}") |
| 440 | print(f"week_file={week_file.as_posix()}") |
| 441 | print(f"week={week}") |
| 442 | run_id = os.environ.get("GITHUB_RUN_ID", "local") |
| 443 | candidate_dir = Path("data/candidates") / week / run_id |
| 444 | print(f"candidate_output_file={(candidate_dir / f'{week}-summary.md').as_posix()}") |
| 445 | print(f"publish_manifest_file={(candidate_dir / 'publish-manifest.json').as_posix()}") |
| 446 | print(f"analysis_gate_report_file={(candidate_dir / 'analysis-gate-report.json').as_posix()}") |
| 447 | print(f"published_output_file=data/analyzed/{week}-summary.md") |
| 448 | PY |
| 449 | ) |
| 450 | { |
| 451 | printf '%s\n' "${CONTEXT_LINES[@]}" |
| 452 | echo "current_datetime=$CURRENT_DATETIME" |
| 453 | echo "run_mode=$RUN_MODE" |
| 454 | echo "source_refresh_policy=$SOURCE_REFRESH_POLICY" |
| 455 | } >> "$GITHUB_OUTPUT" |
| 456 | |
| 457 | - name: Install Python dependencies |
| 458 | run: pip install -r requirements.txt |
| 459 | |
| 460 | - name: Run correlation and press context |
| 461 | id: press-context |
| 462 | env: |
| 463 | IN_WEEK: ${{ steps.analysis-context.outputs.week }} |
| 464 | IN_WEEK_FILE: ${{ steps.analysis-context.outputs.week_file }} |
| 465 | run: | |
| 466 | set -euo pipefail |
| 467 | WEEK="$IN_WEEK" |
| 468 | WEEK_FILE="$IN_WEEK_FILE" |
| 469 | TC_FILE="data/raw/${WEEK}-external-news.json" |
| 470 | if [ ! -f "$TC_FILE" ]; then |
| 471 | TC_FILE="data/raw/${WEEK}-techcrunch.json" |
| 472 | fi |
| 473 | |
| 474 | # Run correlate if external news data exists |
| 475 | if [ -f "$TC_FILE" ]; then |
| 476 | mkdir -p data/analyzed |
| 477 | python scripts/correlate.py \ |
| 478 | --raw "$WEEK_FILE" \ |
| 479 | --techcrunch "$TC_FILE" \ |
| 480 | --output "data/analyzed/${WEEK}-correlations.json" |
| 481 | fi |
| 482 | |
| 483 | # Render press context (handles missing files gracefully) |
| 484 | PRESS_CONTEXT=$(python scripts/render_press_context.py --week "$WEEK") |
| 485 | PRESS_FILE="data/analyzed/${WEEK}-press-context.md" |
| 486 | printf '%s\n' "$PRESS_CONTEXT" > "$PRESS_FILE" |
| 487 | python3 - <<'PY' "$PRESS_FILE" "$GITHUB_OUTPUT" |
| 488 | import sys |
| 489 | from pathlib import Path |
| 490 | from scripts.render_press_context import press_token_estimate |
| 491 | |
| 492 | path = Path(sys.argv[1]) |
| 493 | github_output = Path(sys.argv[2]) |
| 494 | content = path.read_text(encoding="utf-8") |
| 495 | token_estimate = press_token_estimate(content) |
| 496 | print( |
| 497 | "::notice::Press context " |
| 498 | f"size_bytes={path.stat().st_size} " |
| 499 | f"token_estimate={token_estimate}" |
| 500 | ) |
| 501 | with github_output.open("a", encoding="utf-8") as output: |
| 502 | print(f"press_token_estimate={token_estimate}", file=output) |
| 503 | PY |
| 504 | echo "press_file=$PRESS_FILE" >> "$GITHUB_OUTPUT" |
| 505 | |
| 506 | - name: Install Copilot CLI |
| 507 | if: ${{ inputs.analysis_path != 'map-reduce-dry-run' }} |
| 508 | id: install-copilot |
| 509 | run: | |
| 510 | set -euo pipefail |
| 511 | npm install -g @github/copilot |
| 512 | # Verify the CLI actually installed and is runnable before any step |
| 513 | # (including synthesis) is allowed to depend on it. |
| 514 | copilot --version |
| 515 | |
| 516 | - name: Run synthesis step (Step 1) |
| 517 | id: synthesis |
| 518 | env: |
| 519 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 520 | COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GH_TOKEN }} |
| 521 | IN_WEEK_FILE: ${{ steps.analysis-context.outputs.week_file }} |
| 522 | IN_OUTPUT_FILE: ${{ steps.analysis-context.outputs.candidate_output_file }} |
| 523 | IN_CURRENT_DATETIME: ${{ steps.analysis-context.outputs.current_datetime }} |
| 524 | IN_PRESS_FILE: ${{ steps.press-context.outputs.press_file }} |
| 525 | IN_ANALYSIS_PATH: ${{ inputs.analysis_path }} |
| 526 | run: | |
| 527 | set -euo pipefail |
| 528 | WEEK_FILE="$IN_WEEK_FILE" |
| 529 | OUTPUT_FILE="$IN_OUTPUT_FILE" |
| 530 | CURRENT_DATETIME="$IN_CURRENT_DATETIME" |
| 531 | PRESS_FILE="$IN_PRESS_FILE" |
| 532 | DIAGNOSTICS_DIR="$(dirname "$OUTPUT_FILE")/diagnostics" |
| 533 | SYNTHESIS_FILE="$DIAGNOSTICS_DIR/synthesis-narrative.md" |
| 534 | SYNTHESIS_PROMPT="$DIAGNOSTICS_DIR/synthesis-prompt.md" |
| 535 | mkdir -p "$DIAGNOSTICS_DIR" |
| 536 | # Step 1: render synthesis prompt, then run via Copilot CLI. |
| 537 | # Copilot CLI installation/verification (previous step) must have |
| 538 | # already succeeded before this step runs. synthesis_status is a |
| 539 | # precise, fail-closed signal (available/empty/failed/missing) |
| 540 | # recorded on the publish manifest; only "available" is publishable |
| 541 | # for normal-mode publication (see scripts/publish_manifest.py). |
| 542 | python3 scripts/analyze_fallback.py \ |
| 543 | --raw-json "$WEEK_FILE" \ |
| 544 | --output "$OUTPUT_FILE" \ |
| 545 | --current-datetime "$CURRENT_DATETIME" \ |
| 546 | --press-context "$PRESS_FILE" \ |
| 547 | --run-synthesis \ |
| 548 | --synthesis-output "$SYNTHESIS_PROMPT" |
| 549 | SYNTHESIS_STATUS="missing" |
| 550 | if [ ! -f "$SYNTHESIS_PROMPT" ]; then |
| 551 | echo "::warning::Synthesis prompt was not generated; required synthesis is missing." |
| 552 | elif ! command -v copilot >/dev/null 2>&1; then |
| 553 | if [ "$IN_ANALYSIS_PATH" = "map-reduce-dry-run" ]; then |
| 554 | echo "::notice::Copilot CLI intentionally not installed for map-reduce dry run; synthesis skipped (required synthesis is missing)." |
| 555 | else |
| 556 | echo "::error::Copilot CLI is unavailable after installation step; required synthesis is missing." |
| 557 | fi |
| 558 | else |
| 559 | set +e |
| 560 | copilot \ |
| 561 | --agent weekly-synthesis \ |
| 562 | -p "Read the synthesis prompt at ${SYNTHESIS_PROMPT}. Write the compact industry narrative to ${SYNTHESIS_FILE}. Max 2000 tokens. Do not delegate." \ |
| 563 | -s \ |
| 564 | --no-ask-user \ |
| 565 | --allow-tool=read \ |
| 566 | --allow-tool=write \ |
| 567 | > "$DIAGNOSTICS_DIR/synthesis-copilot.log" 2>&1 |
| 568 | SYNTH_STATUS=$? |
| 569 | set -e |
| 570 | if [ "$SYNTH_STATUS" -ne 0 ]; then |
| 571 | SYNTHESIS_STATUS="failed" |
| 572 | echo "::warning::Synthesis Copilot CLI failed (exit=$SYNTH_STATUS); required synthesis is failed." |
| 573 | elif [ ! -s "$SYNTHESIS_FILE" ]; then |
| 574 | SYNTHESIS_STATUS="empty" |
| 575 | echo "::warning::Synthesis Copilot CLI produced no narrative content; required synthesis is empty." |
| 576 | else |
| 577 | SYNTHESIS_STATUS="available" |
| 578 | fi |
| 579 | fi |
| 580 | echo "synthesis_status=$SYNTHESIS_STATUS" >> "$GITHUB_OUTPUT" |
| 581 | if [ "$SYNTHESIS_STATUS" = "available" ]; then |
| 582 | echo "synthesis_file=$SYNTHESIS_FILE" >> "$GITHUB_OUTPUT" |
| 583 | echo "synthesis_available=true" >> "$GITHUB_OUTPUT" |
| 584 | else |
| 585 | echo "synthesis_available=false" >> "$GITHUB_OUTPUT" |
| 586 | fi |
| 587 | |
| 588 | - name: Render and preflight analysis prompt |
| 589 | id: prompt-preflight |
| 590 | env: |
| 591 | IN_WEEK_FILE: ${{ steps.analysis-context.outputs.week_file }} |
| 592 | IN_WEEK: ${{ steps.analysis-context.outputs.week }} |
| 593 | IN_OUTPUT_FILE: ${{ steps.analysis-context.outputs.candidate_output_file }} |
| 594 | IN_CURRENT_DATETIME: ${{ steps.analysis-context.outputs.current_datetime }} |
| 595 | IN_PRESS_FILE: ${{ steps.press-context.outputs.press_file }} |
| 596 | IN_SYNTHESIS_AVAILABLE: ${{ steps.synthesis.outputs.synthesis_available }} |
| 597 | IN_SYNTHESIS_FILE: ${{ steps.synthesis.outputs.synthesis_file }} |
| 598 | run: | |
| 599 | set -euo pipefail |
| 600 | WEEK_FILE="$IN_WEEK_FILE" |
| 601 | WEEK="$IN_WEEK" |
| 602 | OUTPUT_FILE="$IN_OUTPUT_FILE" |
| 603 | CURRENT_DATETIME="$IN_CURRENT_DATETIME" |
| 604 | PRESS_FILE="$IN_PRESS_FILE" |
| 605 | DIAGNOSTICS_DIR="$(dirname "$OUTPUT_FILE")/diagnostics" |
| 606 | PROMPT_FILE="data/metrics/analysis-prompt-${WEEK}.md" |
| 607 | PREFLIGHT_JSON="$DIAGNOSTICS_DIR/analysis-input-manifest.json" |
| 608 | LEGACY_PREFLIGHT_JSON="$DIAGNOSTICS_DIR/analysis-preflight.json" |
| 609 | PREFLIGHT_MD="$DIAGNOSTICS_DIR/analysis-preflight.md" |
| 610 | mkdir -p data/metrics "$(dirname "$OUTPUT_FILE")" "$DIAGNOSTICS_DIR" |
| 611 | # Hydrate metrics ledger from publish before writing this run's prompt/preflight artifacts. |
| 612 | git fetch origin publish 2>/dev/null && \ |
| 613 | git checkout origin/publish -- data/metrics/ 2>/dev/null || true |
| 614 | SYNTHESIS_ARGS=() |
| 615 | if [ "${IN_SYNTHESIS_AVAILABLE:-false}" = "true" ] && [ -f "${IN_SYNTHESIS_FILE:-}" ]; then |
| 616 | SYNTHESIS_ARGS=(--synthesis-input "$IN_SYNTHESIS_FILE") |
| 617 | fi |
| 618 | python3 scripts/analyze_fallback.py \ |
| 619 | --raw-json "$WEEK_FILE" \ |
| 620 | --output "$OUTPUT_FILE" \ |
| 621 | --current-datetime "$CURRENT_DATETIME" \ |
| 622 | --press-context "$PRESS_FILE" \ |
| 623 | --prompt-token-budget "${ANALYSIS_PROMPT_TOKEN_BUDGET:-90000}" \ |
| 624 | --preflight-report-json "$PREFLIGHT_JSON" \ |
| 625 | --preflight-report-md "$PREFLIGHT_MD" \ |
| 626 | "${SYNTHESIS_ARGS[@]}" \ |
| 627 | --print-prompt > "$PROMPT_FILE" |
| 628 | cp "$PREFLIGHT_JSON" "$LEGACY_PREFLIGHT_JSON" |
| 629 | python3 scripts/preflight_cost_check.py \ |
| 630 | --context-files "$PROMPT_FILE" |
| 631 | python3 - <<'PY' "$PREFLIGHT_JSON" |
| 632 | import json |
| 633 | import sys |
| 634 | from pathlib import Path |
| 635 | |
| 636 | report = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) |
| 637 | print( |
| 638 | "::notice::Analysis preflight " |
| 639 | f"tokens={report['prompt_tokens']}/{report['prompt_token_budget']} " |
| 640 | f"bytes={report['prompt_bytes']} " |
| 641 | f"degraded={report['degraded']} " |
| 642 | f"publish_eligible={report['publish_eligible']} " |
| 643 | f"promotion_policy={report.get('promotion_policy', 'unspecified')} " |
| 644 | f"checksum={report['prompt_checksum_sha256'][:12]}" |
| 645 | ) |
| 646 | if report.get("degraded") or not report.get("publish_eligible"): |
| 647 | print( |
| 648 | "::warning::Analysis preflight is degraded/compacted or publish-ineligible; " |
| 649 | "output will remain staged/candidate-only unless an explicit promotion policy allows it." |
| 650 | ) |
| 651 | PY |
| 652 | echo "prompt_file=$PROMPT_FILE" >> "$GITHUB_OUTPUT" |
| 653 | echo "preflight_report_json=$PREFLIGHT_JSON" >> "$GITHUB_OUTPUT" |
| 654 | echo "preflight_report_md=$PREFLIGHT_MD" >> "$GITHUB_OUTPUT" |
| 655 | |
| 656 | - name: Run analysis |
| 657 | if: steps.prompt-preflight.outcome == 'success' |
| 658 | id: run-analysis |
| 659 | env: |
| 660 | COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GH_TOKEN }} |
| 661 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 662 | PREFLIGHT_REPORT_JSON: ${{ steps.prompt-preflight.outputs.preflight_report_json }} |
| 663 | IN_OUTPUT_FILE: ${{ steps.analysis-context.outputs.candidate_output_file }} |
| 664 | IN_WEEK_FILE: ${{ steps.analysis-context.outputs.week_file }} |
| 665 | IN_WEEK: ${{ steps.analysis-context.outputs.week }} |
| 666 | IN_CURRENT_DATETIME: ${{ steps.analysis-context.outputs.current_datetime }} |
| 667 | IN_MANIFEST_FILE: ${{ steps.analysis-context.outputs.publish_manifest_file }} |
| 668 | IN_PUBLISHED_SUMMARY: ${{ steps.analysis-context.outputs.published_output_file }} |
| 669 | IN_PRESS_FILE: ${{ steps.press-context.outputs.press_file }} |
| 670 | IN_PRESS_TOKEN_ESTIMATE: ${{ steps.press-context.outputs.press_token_estimate }} |
| 671 | IN_ANALYSIS_PATH: ${{ inputs.analysis_path || 'single-pass' }} |
| 672 | IN_RUN_MODE: ${{ steps.analysis-context.outputs.run_mode }} |
| 673 | IN_PROMPT_FILE: ${{ steps.prompt-preflight.outputs.prompt_file }} |
| 674 | IN_GATE_REPORT_FILE: ${{ steps.analysis-context.outputs.analysis_gate_report_file }} |
| 675 | run: | |
| 676 | set -euo pipefail |
| 677 | OUTPUT_FILE="$IN_OUTPUT_FILE" |
| 678 | WEEK_FILE="$IN_WEEK_FILE" |
| 679 | WEEK="$IN_WEEK" |
| 680 | CURRENT_DATETIME="$IN_CURRENT_DATETIME" |
| 681 | MANIFEST_FILE="$IN_MANIFEST_FILE" |
| 682 | PUBLISHED_SUMMARY="$IN_PUBLISHED_SUMMARY" |
| 683 | PRESS_FILE="$IN_PRESS_FILE" |
| 684 | PRESS_TOKEN_ESTIMATE="${IN_PRESS_TOKEN_ESTIMATE:-0}" |
| 685 | case "$PRESS_TOKEN_ESTIMATE" in ''|*[!0-9]*) PRESS_TOKEN_ESTIMATE=0 ;; esac |
| 686 | ANALYSIS_PATH="$IN_ANALYSIS_PATH" |
| 687 | RUN_MODE="$IN_RUN_MODE" |
| 688 | ANALYSIS_STARTED=$(date +%s) |
| 689 | DIAGNOSTICS_DIR="$(dirname "$OUTPUT_FILE")/diagnostics" |
| 690 | mkdir -p data/metrics "$(dirname "$OUTPUT_FILE")" "$DIAGNOSTICS_DIR" |
| 691 | PROMPT_FILE="$IN_PROMPT_FILE" |
| 692 | |
| 693 | sanitize_agent_output() { |
| 694 | python3 scripts/sanitize_agent_output.py --path "$1" |
| 695 | } |
| 696 | |
| 697 | run_quality_gate() { |
| 698 | GITHUB_STEP_SUMMARY="" python3 scripts/analysis_gate.py \ |
| 699 | --analysis-file "$OUTPUT_FILE" \ |
| 700 | --raw-json "$WEEK_FILE" \ |
| 701 | --current-datetime "$CURRENT_DATETIME" \ |
| 702 | --source "$1" \ |
| 703 | --model "$2" \ |
| 704 | --repair-safe \ |
| 705 | --press-context-path "$PRESS_FILE" \ |
| 706 | --press-token-estimate "$PRESS_TOKEN_ESTIMATE" \ |
| 707 | --report-json "$3" |
| 708 | } |
| 709 | |
| 710 | if [ "$ANALYSIS_PATH" = "map-reduce-dry-run" ]; then |
| 711 | if [ "$RUN_MODE" != "dry-run" ] && [ "$RUN_MODE" != "candidate-only" ]; then |
| 712 | echo "::error::map-reduce-dry-run requires run_mode=dry-run or candidate-only and cannot run in a publishing mode." |
| 713 | exit 1 |
| 714 | fi |
| 715 | MAP_REDUCE_DIR="$(dirname "$OUTPUT_FILE")/map-reduce" |
| 716 | python3 scripts/map_reduce_dry_run.py \ |
| 717 | --raw-json "$WEEK_FILE" \ |
| 718 | --press-context "$PRESS_FILE" \ |
| 719 | --output-dir "$MAP_REDUCE_DIR" \ |
| 720 | --current-datetime "$CURRENT_DATETIME" \ |
| 721 | --run-id "${GITHUB_RUN_ID:-local}" \ |
| 722 | --baseline-summary "$PUBLISHED_SUMMARY" |
| 723 | cp "$MAP_REDUCE_DIR/${WEEK}-map-reduce-candidate.md" "$OUTPUT_FILE" |
| 724 | ANALYSIS_SOURCE="map-reduce-dry-run" |
| 725 | ANALYSIS_MODEL="local-deterministic" |
| 726 | python3 scripts/track_token_usage.py \ |
| 727 | --stage analysis \ |
| 728 | --source "$ANALYSIS_SOURCE" \ |
| 729 | --model "$ANALYSIS_MODEL" \ |
| 730 | --current-datetime "$CURRENT_DATETIME" \ |
| 731 | --week "$WEEK" \ |
| 732 | --prompt-file "$PROMPT_FILE" \ |
| 733 | --output-file "$OUTPUT_FILE" \ |
| 734 | --input-manifest "$PREFLIGHT_REPORT_JSON" |
| 735 | rm -f "$PROMPT_FILE" |
| 736 | echo "analysis_source=$ANALYSIS_SOURCE" >> "$GITHUB_OUTPUT" |
| 737 | echo "analysis_model=$ANALYSIS_MODEL" >> "$GITHUB_OUTPUT" |
| 738 | exit 0 |
| 739 | fi |
| 740 | |
| 741 | # Retry loop: LLM output can be non-deterministically truncated, |
| 742 | # so retry up to 3 attempts if the quality gate rejects the article. |
| 743 | MAX_RETRIES=2 |
| 744 | ATTEMPT=0 |
| 745 | GATE_PASSED=false |
| 746 | ANALYSIS_SOURCE="" |
| 747 | ANALYSIS_MODEL="" |
| 748 | LAST_GATE_FINGERPRINT="" |
| 749 | FINAL_FAILURE_CLASS="" |
| 750 | |
| 751 | if command -v copilot >/dev/null 2>&1; then |
| 752 | while [ "$GATE_PASSED" = "false" ] && [ "$ATTEMPT" -le "$MAX_RETRIES" ]; do |
| 753 | if [ "$ATTEMPT" -gt 0 ]; then |
| 754 | echo "::warning::Quality gate failed on attempt $ATTEMPT; retrying with focused gate diagnostics (attempt $((ATTEMPT + 1))/$((MAX_RETRIES + 1)))..." |
| 755 | rm -f "$OUTPUT_FILE" |
| 756 | # Reset any .squad changes from failed attempt |
| 757 | git checkout -- .squad 2>/dev/null || true |
| 758 | fi |
| 759 | |
| 760 | echo "::notice::Running Copilot analysis attempt $((ATTEMPT + 1))/$((MAX_RETRIES + 1))" |
| 761 | TRANSCRIPT_FILE="data/metrics/copilot-transcript-attempt-${ATTEMPT}.md" |
| 762 | GATE_REPORT="$DIAGNOSTICS_DIR/gate-copilot-cli-attempt-${ATTEMPT}.json" |
| 763 | CANDIDATE_SNAPSHOT="$DIAGNOSTICS_DIR/candidate-copilot-cli-attempt-${ATTEMPT}.md" |
| 764 | COPILOT_LOG="$DIAGNOSTICS_DIR/copilot-cli-attempt-${ATTEMPT}.log" |
| 765 | COPILOT_FAILURE_REPORT="$DIAGNOSTICS_DIR/copilot-cli-failure-attempt-${ATTEMPT}.json" |
| 766 | rm -f "$TRANSCRIPT_FILE" "$COPILOT_LOG" "$COPILOT_FAILURE_REPORT" |
| 767 | REPAIR_CONTEXT="" |
| 768 | if [ "$ATTEMPT" -gt 0 ]; then |
| 769 | PREVIOUS_REPORT="$DIAGNOSTICS_DIR/gate-copilot-cli-attempt-$((ATTEMPT - 1)).json" |
| 770 | if [ -f "$PREVIOUS_REPORT" ]; then |
| 771 | REPAIR_CONTEXT=" Previous gate report: ${PREVIOUS_REPORT}. Correct exactly those validation errors; do not regenerate unrelated content." |
| 772 | fi |
| 773 | fi |
| 774 | |
| 775 | # Model is configured in .github/agents/weekly-analysis.agent.md (gpt-5.5) |
| 776 | set +e |
| 777 | copilot \ |
| 778 | --agent weekly-analysis \ |
| 779 | -p "Read the file at ${PROMPT_FILE}. Write the complete weekly analysis markdown to ${OUTPUT_FILE}. The first bytes must be --- and the run is incomplete until ${OUTPUT_FILE} exists and is non-empty. Do not delegate. Do not spawn sub-agents. Do not emit commentary.${REPAIR_CONTEXT}" \ |
| 780 | -s \ |
| 781 | --no-ask-user \ |
| 782 | --allow-tool=read \ |
| 783 | --allow-tool=write \ |
| 784 | --share="$TRANSCRIPT_FILE" \ |
| 785 | > "$COPILOT_LOG" 2>&1 |
| 786 | COPILOT_STATUS=$? |
| 787 | set -e |
| 788 | if [ "$COPILOT_STATUS" -ne 0 ]; then |
| 789 | FAILURE_CLASS=$(python3 scripts/copilot_failure.py \ |
| 790 | --log "$COPILOT_LOG" \ |
| 791 | --exit-code "$COPILOT_STATUS" \ |
| 792 | --report-json "$COPILOT_FAILURE_REPORT" \ |
| 793 | --create-token-issue \ |
| 794 | --repo "${GITHUB_REPOSITORY:-jmservera/SquadScope}" \ |
| 795 | --assignee "jmservera" \ |
| 796 | --week "$WEEK" \ |
| 797 | --run-id "${GITHUB_RUN_ID:-local}") |
| 798 | FINAL_FAILURE_CLASS="$FAILURE_CLASS" |
| 799 | echo "::warning::Copilot CLI failed on attempt $((ATTEMPT + 1)); class=${FAILURE_CLASS}; report=${COPILOT_FAILURE_REPORT}" |
| 800 | if [ "$FAILURE_CLASS" = "copilot_token_failure" ] || [ "$FAILURE_CLASS" = "copilot_inaccessible" ]; then |
| 801 | echo "::error::Copilot access failure (${FAILURE_CLASS}). Renewal issue was created or updated; failing without no-AI fallback. See ${COPILOT_FAILURE_REPORT}." |
| 802 | exit 1 |
| 803 | fi |
| 804 | if [ "$FAILURE_CLASS" = "context_too_large" ]; then |
| 805 | echo "::warning::Non-retryable Copilot analysis failure (${FAILURE_CLASS}). Writing a non-publishable no-AI candidate so the current published article can be preserved. See ${COPILOT_FAILURE_REPORT}." |
| 806 | break |
| 807 | fi |
| 808 | ATTEMPT=$((ATTEMPT + 1)) |
| 809 | continue |
| 810 | fi |
| 811 | |
| 812 | if ! test -s "$OUTPUT_FILE"; then |
| 813 | FINAL_FAILURE_CLASS="writer_contract_failure" |
| 814 | echo "::warning::Copilot analysis completed without writing ${OUTPUT_FILE}; class=${FINAL_FAILURE_CLASS}" |
| 815 | ATTEMPT=$((ATTEMPT + 1)) |
| 816 | continue |
| 817 | fi |
| 818 | |
| 819 | FINAL_FAILURE_CLASS="" |
| 820 | sanitize_agent_output "$OUTPUT_FILE" |
| 821 | |
| 822 | # Inline quality gate check (suppress step summary to avoid noise). |
| 823 | # The gate applies deterministic metadata/schema repairs and writes an auditable report. |
| 824 | if run_quality_gate copilot-cli copilot-default "$GATE_REPORT"; then |
| 825 | cp "$OUTPUT_FILE" "$CANDIDATE_SNAPSHOT" 2>/dev/null || true |
| 826 | cp "$GATE_REPORT" "$IN_GATE_REPORT_FILE" 2>/dev/null || true |
| 827 | GATE_PASSED=true |
| 828 | FINAL_TRANSCRIPT="$TRANSCRIPT_FILE" |
| 829 | ANALYSIS_SOURCE="copilot-cli" |
| 830 | ANALYSIS_MODEL="copilot-default" |
| 831 | else |
| 832 | cp "$OUTPUT_FILE" "$CANDIDATE_SNAPSHOT" 2>/dev/null || true |
| 833 | CURRENT_GATE_FINGERPRINT=$(python3 -c 'import sys; from pathlib import Path; import scripts.analysis_gate as gate; print(gate.gate_report_fingerprint(Path(sys.argv[1])))' "$GATE_REPORT" 2>/dev/null || true) |
| 834 | if [ -n "$CURRENT_GATE_FINGERPRINT" ]; then |
| 835 | if [ -n "$LAST_GATE_FINGERPRINT" ] && [ "$CURRENT_GATE_FINGERPRINT" = "$LAST_GATE_FINGERPRINT" ]; then |
| 836 | echo "::error::Repeated deterministic analysis gate failure after repair. See ${GATE_REPORT} and ${CANDIDATE_SNAPSHOT}." |
| 837 | break |
| 838 | fi |
| 839 | LAST_GATE_FINGERPRINT="$CURRENT_GATE_FINGERPRINT" |
| 840 | else |
| 841 | echo "::warning::Gate report missing or invalid; continuing fallback path without deterministic repeat fingerprint. Expected report: ${GATE_REPORT}" |
| 842 | fi |
| 843 | fi |
| 844 | |
| 845 | ATTEMPT=$((ATTEMPT + 1)) |
| 846 | done |
| 847 | else |
| 848 | FINAL_FAILURE_CLASS="copilot_inaccessible" |
| 849 | COPILOT_LOG="$DIAGNOSTICS_DIR/copilot-cli-unavailable.log" |
| 850 | COPILOT_FAILURE_REPORT="$DIAGNOSTICS_DIR/copilot-cli-failure-unavailable.json" |
| 851 | echo "copilot is not available: command not found" > "$COPILOT_LOG" |
| 852 | python3 scripts/copilot_failure.py \ |
| 853 | --log "$COPILOT_LOG" \ |
| 854 | --exit-code 127 \ |
| 855 | --report-json "$COPILOT_FAILURE_REPORT" \ |
| 856 | --create-token-issue \ |
| 857 | --repo "${GITHUB_REPOSITORY:-jmservera/SquadScope}" \ |
| 858 | --assignee "jmservera" \ |
| 859 | --week "$WEEK" \ |
| 860 | --run-id "${GITHUB_RUN_ID:-local}" >/dev/null |
| 861 | echo "::error::Copilot CLI unavailable. Renewal issue was created or updated; failing without no-AI fallback. See ${COPILOT_FAILURE_REPORT}." |
| 862 | exit 1 |
| 863 | fi |
| 864 | |
| 865 | if [ "$GATE_PASSED" = "false" ]; then |
| 866 | echo "::warning::No publishable Copilot summary was produced. Final failure class: ${FINAL_FAILURE_CLASS:-quality_gate}. The publish manifest will preserve any existing good weekly article." |
| 867 | python3 scripts/analyze_fallback.py \ |
| 868 | --raw-json "$WEEK_FILE" \ |
| 869 | --output "$OUTPUT_FILE" \ |
| 870 | --current-datetime "$CURRENT_DATETIME" \ |
| 871 | --press-context "$PRESS_FILE" \ |
| 872 | --no-ai |
| 873 | sanitize_agent_output "$OUTPUT_FILE" |
| 874 | NO_AI_GATE_REPORT="$DIAGNOSTICS_DIR/gate-no-ai-attempt-0.json" |
| 875 | run_quality_gate no-ai none "$NO_AI_GATE_REPORT" || true |
| 876 | cp "$NO_AI_GATE_REPORT" "$IN_GATE_REPORT_FILE" 2>/dev/null || true |
| 877 | cp "$OUTPUT_FILE" "$DIAGNOSTICS_DIR/candidate-no-ai-attempt-0.md" 2>/dev/null || true |
| 878 | ANALYSIS_SOURCE="no-ai" |
| 879 | ANALYSIS_MODEL="none" |
| 880 | fi |
| 881 | |
| 882 | # Copy final transcript to canonical location |
| 883 | if [ -n "${FINAL_TRANSCRIPT:-}" ]; then |
| 884 | cp "$FINAL_TRANSCRIPT" data/metrics/copilot-transcript.md 2>/dev/null || true |
| 885 | else |
| 886 | rm -f data/metrics/copilot-transcript.md |
| 887 | fi |
| 888 | |
| 889 | TRANSCRIPT_ARGS="" |
| 890 | if [ -f "data/metrics/copilot-transcript.md" ]; then |
| 891 | TRANSCRIPT_ARGS="--transcript data/metrics/copilot-transcript.md" |
| 892 | fi |
| 893 | |
| 894 | python3 scripts/track_token_usage.py \ |
| 895 | --stage analysis \ |
| 896 | --source "$ANALYSIS_SOURCE" \ |
| 897 | --model "$ANALYSIS_MODEL" \ |
| 898 | --current-datetime "$CURRENT_DATETIME" \ |
| 899 | --week "$WEEK" \ |
| 900 | --prompt-file "$PROMPT_FILE" \ |
| 901 | --output-file "$OUTPUT_FILE" \ |
| 902 | --input-manifest "$PREFLIGHT_REPORT_JSON" \ |
| 903 | $TRANSCRIPT_ARGS |
| 904 | ANALYSIS_DURATION=$(( $(date +%s) - ANALYSIS_STARTED )) |
| 905 | echo "::notice::Analysis path source=$ANALYSIS_SOURCE model=$ANALYSIS_MODEL duration_seconds=$ANALYSIS_DURATION press_context=$PRESS_FILE" |
| 906 | rm -f "$PROMPT_FILE" |
| 907 | # Clean up per-attempt transcripts |
| 908 | rm -f data/metrics/copilot-transcript-attempt-*.md |
| 909 | echo "analysis_source=$ANALYSIS_SOURCE" >> "$GITHUB_OUTPUT" |
| 910 | echo "analysis_model=$ANALYSIS_MODEL" >> "$GITHUB_OUTPUT" |
| 911 | |
| 912 | - name: budget-alerts |
| 913 | if: always() |
| 914 | run: | |
| 915 | python3 scripts/budget_alerts.py --metrics data/metrics/token-usage.jsonl |
| 916 | |
| 917 | - name: quality-check |
| 918 | id: quality-check |
| 919 | continue-on-error: true |
| 920 | env: |
| 921 | ANALYSIS_FILE: ${{ steps.analysis-context.outputs.candidate_output_file }} |
| 922 | ANALYSIS_SOURCE: ${{ steps.run-analysis.outputs.analysis_source }} |
| 923 | ANALYSIS_MODEL: ${{ steps.run-analysis.outputs.analysis_model }} |
| 924 | RAW_JSON_FILE: ${{ steps.analysis-context.outputs.week_file }} |
| 925 | CURRENT_DATETIME: ${{ steps.analysis-context.outputs.current_datetime }} |
| 926 | GATE_REPORT: ${{ steps.analysis-context.outputs.analysis_gate_report_file }} |
| 927 | PRESS_FILE: ${{ steps.press-context.outputs.press_file }} |
| 928 | IN_PRESS_TOKEN_ESTIMATE: ${{ steps.press-context.outputs.press_token_estimate }} |
| 929 | run: | |
| 930 | set -euo pipefail |
| 931 | IN_PRESS_TOKEN_ESTIMATE="${IN_PRESS_TOKEN_ESTIMATE:-0}" |
| 932 | case "$IN_PRESS_TOKEN_ESTIMATE" in ''|*[!0-9]*) IN_PRESS_TOKEN_ESTIMATE=0 ;; esac |
| 933 | python3 scripts/analysis_gate.py \ |
| 934 | --analysis-file "$ANALYSIS_FILE" \ |
| 935 | --raw-json "$RAW_JSON_FILE" \ |
| 936 | --current-datetime "$CURRENT_DATETIME" \ |
| 937 | --source "$ANALYSIS_SOURCE" \ |
| 938 | --model "$ANALYSIS_MODEL" \ |
| 939 | --press-context-path "$PRESS_FILE" \ |
| 940 | --press-token-estimate "$IN_PRESS_TOKEN_ESTIMATE" \ |
| 941 | --report-json "$GATE_REPORT" |
| 942 | |
| 943 | - name: Emit publish eligibility manifest |
| 944 | env: |
| 945 | WEEK: ${{ steps.analysis-context.outputs.week }} |
| 946 | RUN_ID: ${{ github.run_id }} |
| 947 | CURRENT_DATETIME: ${{ steps.analysis-context.outputs.current_datetime }} |
| 948 | CANDIDATE_SUMMARY: ${{ steps.analysis-context.outputs.candidate_output_file }} |
| 949 | PUBLISHED_SUMMARY: ${{ steps.analysis-context.outputs.published_output_file }} |
| 950 | RAW_JSON_FILE: ${{ steps.analysis-context.outputs.week_file }} |
| 951 | ANALYSIS_SOURCE: ${{ steps.run-analysis.outputs.analysis_source }} |
| 952 | ANALYSIS_MODEL: ${{ steps.run-analysis.outputs.analysis_model }} |
| 953 | VALIDATION_STATUS: ${{ steps.quality-check.outcome == 'success' && 'passed' || 'failed' }} |
| 954 | MANIFEST_FILE: ${{ steps.analysis-context.outputs.publish_manifest_file }} |
| 955 | RUN_MODE: ${{ steps.analysis-context.outputs.run_mode }} |
| 956 | SOURCE_REFRESH_POLICY: ${{ steps.analysis-context.outputs.source_refresh_policy }} |
| 957 | SOURCE_RUN_ID: ${{ inputs.source_run_id || '' }} |
| 958 | GATE_REPORT: ${{ steps.analysis-context.outputs.analysis_gate_report_file }} |
| 959 | PREFLIGHT_REPORT: ${{ steps.prompt-preflight.outputs.preflight_report_json }} |
| 960 | SYNTHESIS_STATUS: ${{ steps.synthesis.outputs.synthesis_status }} |
| 961 | SYNTHESIS_FILE: ${{ steps.synthesis.outputs.synthesis_file }} |
| 962 | FORCE_REASON: ${{ inputs.force_reason || '' }} |
| 963 | GH_ACTOR: ${{ github.actor }} |
| 964 | run: | |
| 965 | set -euo pipefail |
| 966 | git fetch origin publish 2>/dev/null && git checkout origin/publish -- "$PUBLISHED_SUMMARY" 2>/dev/null || true |
| 967 | ARTIFACT_ARGS=() |
| 968 | RESTORE_ARGS=() |
| 969 | if [ "$RUN_MODE" = "restore" ]; then |
| 970 | RESTORE_ARGS=( |
| 971 | --source-run-id "$SOURCE_RUN_ID" |
| 972 | --raw-store-manifest "data/raw-store/${WEEK}/${SOURCE_RUN_ID}/manifest.json" |
| 973 | ) |
| 974 | fi |
| 975 | for candidate in \ |
| 976 | "external_news=data/raw/${WEEK}-external-news.json" \ |
| 977 | "techcrunch_news=data/raw/${WEEK}-techcrunch.json" \ |
| 978 | "correlations=data/analyzed/${WEEK}-correlations.json" \ |
| 979 | "press_context=data/analyzed/${WEEK}-press-context.md" \ |
| 980 | "map_reduce_manifest=data/candidates/${WEEK}/${RUN_ID}/map-reduce/manifest.json" \ |
| 981 | "map_reduce_qa=data/candidates/${WEEK}/${RUN_ID}/map-reduce/qa-comparison-report.json" |
| 982 | do |
| 983 | path="${candidate#*=}" |
| 984 | [ -f "$path" ] && ARTIFACT_ARGS+=(--artifact "$candidate") |
| 985 | done |
| 986 | SYNTHESIS_ARGS=(--synthesis-status "${SYNTHESIS_STATUS:-missing}") |
| 987 | [ -n "${SYNTHESIS_FILE:-}" ] && [ -f "${SYNTHESIS_FILE:-}" ] && SYNTHESIS_ARGS+=(--synthesis-file "$SYNTHESIS_FILE") |
| 988 | FORCE_ARGS=() |
| 989 | if [ -n "$FORCE_REASON" ]; then |
| 990 | FORCE_ARGS=(--publish-policy force-replace --force-reason "$FORCE_REASON" --actor "$GH_ACTOR") |
| 991 | fi |
| 992 | python3 scripts/publish_manifest.py create \ |
| 993 | --week "$WEEK" \ |
| 994 | --run-id "$RUN_ID" \ |
| 995 | --current-datetime "$CURRENT_DATETIME" \ |
| 996 | --summary "$CANDIDATE_SUMMARY" \ |
| 997 | --published-summary "$PUBLISHED_SUMMARY" \ |
| 998 | --raw-json "$RAW_JSON_FILE" \ |
| 999 | --analysis-source "$ANALYSIS_SOURCE" \ |
| 1000 | --analysis-model "$ANALYSIS_MODEL" \ |
| 1001 | --preflight-report "$PREFLIGHT_REPORT" \ |
| 1002 | --validation-status "$VALIDATION_STATUS" \ |
| 1003 | --run-mode "$RUN_MODE" \ |
| 1004 | --source-refresh-policy "$SOURCE_REFRESH_POLICY" \ |
| 1005 | --gate-report "$GATE_REPORT" \ |
| 1006 | "${SYNTHESIS_ARGS[@]}" \ |
| 1007 | --output "$MANIFEST_FILE" \ |
| 1008 | "${RESTORE_ARGS[@]}" \ |
| 1009 | "${FORCE_ARGS[@]}" \ |
| 1010 | "${ARTIFACT_ARGS[@]}" |
| 1011 | |
| 1012 | - name: Assert candidate is eligible for promotion |
| 1013 | if: ${{ steps.analysis-context.outputs.run_mode != 'dry-run' && steps.analysis-context.outputs.run_mode != 'candidate-only' }} |
| 1014 | env: |
| 1015 | IN_MANIFEST_FILE: ${{ steps.analysis-context.outputs.publish_manifest_file }} |
| 1016 | run: python3 scripts/publish_manifest.py assert-eligible --manifest "$IN_MANIFEST_FILE" |
| 1017 | |
| 1018 | - name: Upload analyzed data |
| 1019 | if: always() |
| 1020 | uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| 1021 | with: |
| 1022 | name: analyzed-data |
| 1023 | path: data/analyzed/ |
| 1024 | if-no-files-found: warn |
| 1025 | |
| 1026 | - name: Upload analysis candidate |
| 1027 | if: always() |
| 1028 | uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| 1029 | with: |
| 1030 | name: analysis-candidate |
| 1031 | path: data/candidates/ |
| 1032 | if-no-files-found: warn |
| 1033 | |
| 1034 | generate: |
| 1035 | needs: analyze |
| 1036 | if: ${{ needs.analyze.outputs.run_mode != 'dry-run' && needs.analyze.outputs.run_mode != 'candidate-only' }} |
| 1037 | runs-on: ubuntu-latest |
| 1038 | permissions: |
| 1039 | actions: read |
| 1040 | contents: write |
| 1041 | outputs: |
| 1042 | page_path: ${{ steps.generate-content.outputs.page_path }} |
| 1043 | |
| 1044 | steps: |
| 1045 | - name: Check out repository # zizmor: ignore[artipacked] generate job pushes to the publish branch; checkout token is reused by a later git push |
| 1046 | uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 |
| 1047 | with: |
| 1048 | fetch-depth: 0 |
| 1049 | ref: ${{ github.event.repository.default_branch }} |
| 1050 | |
| 1051 | - name: Download analyzed data artifact |
| 1052 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1053 | with: |
| 1054 | name: analyzed-data |
| 1055 | path: data/analyzed/ |
| 1056 | |
| 1057 | - name: Download analysis candidate artifact |
| 1058 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1059 | with: |
| 1060 | name: analysis-candidate |
| 1061 | path: data/candidates/ |
| 1062 | |
| 1063 | - name: Download raw crawl artifact |
| 1064 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1065 | with: |
| 1066 | name: raw-data |
| 1067 | path: data/raw/ |
| 1068 | |
| 1069 | - name: Set up Python |
| 1070 | uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 |
| 1071 | with: |
| 1072 | python-version: '3.12' |
| 1073 | |
| 1074 | - name: Install Python dependencies |
| 1075 | run: pip install -r requirements.txt |
| 1076 | |
| 1077 | - name: Generate weekly content |
| 1078 | id: generate-content |
| 1079 | env: |
| 1080 | WEEK: ${{ needs.analyze.outputs.week }} |
| 1081 | SUMMARY_FILE: ${{ needs.analyze.outputs.candidate_summary_file }} |
| 1082 | MANIFEST_FILE: ${{ needs.analyze.outputs.publish_manifest_file }} |
| 1083 | run: | |
| 1084 | set -euo pipefail |
| 1085 | python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE" |
| 1086 | python3 - <<'PYGEN' "$WEEK" "$SUMMARY_FILE" "$MANIFEST_FILE" >> "$GITHUB_OUTPUT" |
| 1087 | import json |
| 1088 | import sys |
| 1089 | from pathlib import Path |
| 1090 | |
| 1091 | import scripts.generate_content as generate_content |
| 1092 | import scripts.publish_manifest as publish_manifest |
| 1093 | |
| 1094 | week = sys.argv[1] |
| 1095 | summary_path = Path(sys.argv[2]) |
| 1096 | manifest_path = Path(sys.argv[3]) |
| 1097 | candidate_content = manifest_path.parent / f"{week}-content.md" |
| 1098 | page_path = generate_content.generate_content(summary_path, candidate_content) |
| 1099 | |
| 1100 | manifest = json.loads(manifest_path.read_text(encoding="utf-8")) |
| 1101 | manifest["candidate_content_path"] = page_path.as_posix() |
| 1102 | manifest.setdefault("candidate", {})["content_path"] = page_path.as_posix() |
| 1103 | manifest["candidate"]["content_sha256"] = publish_manifest.sha256_file(page_path) |
| 1104 | manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") |
| 1105 | |
| 1106 | final_page_path = generate_content.infer_output_path(week, Path.cwd()) |
| 1107 | print(f"page_path={final_page_path.as_posix()}") |
| 1108 | print(f"candidate_content_path={page_path.as_posix()}") |
| 1109 | PYGEN |
| 1110 | python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE" |
| 1111 | python3 scripts/promotion_guard.py --manifest "$MANIFEST_FILE" |
| 1112 | |
| 1113 | - name: Hydrate analyzed data from publish |
| 1114 | env: |
| 1115 | WEEK: ${{ needs.analyze.outputs.week }} |
| 1116 | MANIFEST_FILE: ${{ needs.analyze.outputs.publish_manifest_file }} |
| 1117 | run: | |
| 1118 | set -euo pipefail |
| 1119 | python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE" |
| 1120 | git fetch origin publish |
| 1121 | # Restore prior weeks' analyzed files from publish so generate_rollups.py |
| 1122 | # sees accurate historical data. Skip the current week's file (already |
| 1123 | # present from the downloaded artifact). |
| 1124 | git ls-tree -r --name-only origin/publish -- data/analyzed/ 2>/dev/null | while read -r f; do |
| 1125 | case "$f" in |
| 1126 | *"${WEEK}"*) continue ;; |
| 1127 | esac |
| 1128 | git checkout origin/publish -- "$f" 2>/dev/null || true |
| 1129 | done |
| 1130 | |
| 1131 | - name: Generate rollups |
| 1132 | run: python3 scripts/generate_rollups.py |
| 1133 | |
| 1134 | - name: Commit generated content to data branch |
| 1135 | env: |
| 1136 | DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} |
| 1137 | DATA_BRANCH: publish |
| 1138 | WEEK: ${{ needs.analyze.outputs.week }} |
| 1139 | PAGE_PATH: ${{ steps.generate-content.outputs.page_path }} |
| 1140 | MANIFEST_FILE: ${{ needs.analyze.outputs.publish_manifest_file }} |
| 1141 | EXPECTED_PUBLISH_SHA: ${{ needs.analyze.outputs.publish_head_sha }} |
| 1142 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 1143 | run: | |
| 1144 | set -euo pipefail |
| 1145 | python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE" |
| 1146 | cp scripts/publish_safety.py publish-safety-tool.py |
| 1147 | cp scripts/promotion_guard.py promotion-guard-tool.py |
| 1148 | git config user.name "github-actions[bot]" |
| 1149 | git config user.email "github-actions[bot]@users.noreply.github.com" |
| 1150 | python3 scripts/promotion_guard.py --manifest "$MANIFEST_FILE" |
| 1151 | python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE" |
| 1152 | if ! git status --short -- data/analyzed data/candidates data/metrics data/published content/weekly content/monthly content/yearly | grep -q .; then |
| 1153 | echo "No promoted analysis or generated content changes to commit." |
| 1154 | rm -f publish-safety-tool.py promotion-guard-tool.py |
| 1155 | exit 0 |
| 1156 | fi |
| 1157 | cp -r data/analyzed analyzed-data-backup |
| 1158 | cp -r data/candidates candidates-data-backup |
| 1159 | cp -r data/metrics metrics-data-backup 2>/dev/null || true |
| 1160 | cp -r data/published published-data-backup 2>/dev/null || true |
| 1161 | cp -r content/weekly content-weekly-backup 2>/dev/null || true |
| 1162 | cp -r content/monthly content-monthly-backup 2>/dev/null || true |
| 1163 | cp -r content/yearly content-yearly-backup 2>/dev/null || true |
| 1164 | # Push to the unprotected data branch |
| 1165 | if git fetch origin "$DATA_BRANCH" 2>/dev/null; then |
| 1166 | CURRENT_PUBLISH_SHA=$(git rev-parse "origin/$DATA_BRANCH") |
| 1167 | if [ -n "$EXPECTED_PUBLISH_SHA" ] && [ "$CURRENT_PUBLISH_SHA" != "$EXPECTED_PUBLISH_SHA" ]; then |
| 1168 | echo "::error::Publish branch drifted between analyze and content promotion: expected $EXPECTED_PUBLISH_SHA, found $CURRENT_PUBLISH_SHA" |
| 1169 | exit 1 |
| 1170 | fi |
| 1171 | git checkout -f -B "$DATA_BRANCH" "origin/$DATA_BRANCH" |
| 1172 | else |
| 1173 | CURRENT_PUBLISH_SHA="" |
| 1174 | if [ -n "$EXPECTED_PUBLISH_SHA" ]; then |
| 1175 | echo "::error::Publish branch disappeared between analyze and content promotion: expected $EXPECTED_PUBLISH_SHA" |
| 1176 | exit 1 |
| 1177 | fi |
| 1178 | git fetch origin "$DEFAULT_BRANCH" |
| 1179 | git checkout -f -B "$DATA_BRANCH" "origin/$DEFAULT_BRANCH" |
| 1180 | fi |
| 1181 | mkdir -p data/analyzed data/candidates data/metrics data/published content/weekly content/monthly content/yearly |
| 1182 | # generate_content.py returns an absolute path; normalize it before restoring |
| 1183 | # the freshly generated page onto the publish branch checkout. |
| 1184 | case "$PAGE_PATH" in |
| 1185 | "$GITHUB_WORKSPACE"/*) PAGE_PATH="${PAGE_PATH#"$GITHUB_WORKSPACE"/}" ;; |
| 1186 | esac |
| 1187 | PAGE_PATH="${PAGE_PATH#/}" |
| 1188 | case "$PAGE_PATH" in |
| 1189 | content/weekly/*) ;; |
| 1190 | *) |
| 1191 | echo "::error::Expected PAGE_PATH under content/weekly/, got: $PAGE_PATH" |
| 1192 | exit 1 |
| 1193 | ;; |
| 1194 | esac |
| 1195 | RELATIVE_FROM_WEEKLY="${PAGE_PATH#content/weekly/}" |
| 1196 | mkdir -p "$(dirname "$PAGE_PATH")" |
| 1197 | cp -r "candidates-data-backup/${WEEK}" data/candidates/ |
| 1198 | python3 publish-safety-tool.py backup-existing \ |
| 1199 | --week "$WEEK" \ |
| 1200 | --run-id "$GITHUB_RUN_ID" \ |
| 1201 | --kind content \ |
| 1202 | --manifest "$MANIFEST_FILE" \ |
| 1203 | --expected-publish-ref "$EXPECTED_PUBLISH_SHA" \ |
| 1204 | --actual-publish-ref "$CURRENT_PUBLISH_SHA" \ |
| 1205 | --path "data/analyzed/${WEEK}-summary.md" \ |
| 1206 | --path "data/analyzed/${WEEK}-correlations.json" \ |
| 1207 | --path "data/analyzed/${WEEK}-press-context.md" \ |
| 1208 | --path "data/published/${WEEK}/promotion-manifest.json" \ |
| 1209 | --path "$PAGE_PATH" |
| 1210 | cp -r metrics-data-backup/* data/metrics/ 2>/dev/null || true |
| 1211 | cp "analyzed-data-backup/${WEEK}-correlations.json" data/analyzed/ 2>/dev/null || true |
| 1212 | cp "analyzed-data-backup/${WEEK}-press-context.md" data/analyzed/ 2>/dev/null || true |
| 1213 | python3 promotion-guard-tool.py --manifest "data/candidates/${WEEK}/${GITHUB_RUN_ID}/publish-manifest.json" |
| 1214 | # Monthly/yearly rollups are safe to copy since generate_rollups.py was seeded |
| 1215 | # with hydrated prior-week data (see Hydrate analyzed data step above) |
| 1216 | cp -r content-monthly-backup/* content/monthly/ 2>/dev/null || true |
| 1217 | cp -r content-yearly-backup/* content/yearly/ 2>/dev/null || true |
| 1218 | rm -rf analyzed-data-backup candidates-data-backup metrics-data-backup published-data-backup content-weekly-backup content-monthly-backup content-yearly-backup publish-safety-tool.py promotion-guard-tool.py |
| 1219 | git add data/analyzed/ data/candidates/ data/metrics/ data/published/ content/weekly/ content/monthly/ content/yearly/ data/backups/ |
| 1220 | git diff --cached --quiet && exit 0 |
| 1221 | git commit -m "publish: weekly article transaction $WEEK [run #${GITHUB_RUN_ID}]" |
| 1222 | if [ -n "$CURRENT_PUBLISH_SHA" ]; then |
| 1223 | git push --force-with-lease="refs/heads/$DATA_BRANCH:$CURRENT_PUBLISH_SHA" origin HEAD:"$DATA_BRANCH" |
| 1224 | else |
| 1225 | git push origin HEAD:"$DATA_BRANCH" |
| 1226 | fi |
| 1227 | |
| 1228 | - name: Upload generated content artifact |
| 1229 | uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| 1230 | with: |
| 1231 | name: generated-content |
| 1232 | path: | |
| 1233 | content/weekly/ |
| 1234 | content/monthly/ |
| 1235 | content/yearly/ |
| 1236 | if-no-files-found: warn |
| 1237 | |
| 1238 | - name: Upload promoted analyzed artifact |
| 1239 | uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 |
| 1240 | with: |
| 1241 | name: promoted-analyzed-data |
| 1242 | path: data/analyzed/ |
| 1243 | if-no-files-found: warn |
| 1244 | |
| 1245 | deploy: |
| 1246 | needs: [crawl, analyze, generate] |
| 1247 | if: ${{ needs.analyze.outputs.run_mode != 'dry-run' && needs.analyze.outputs.run_mode != 'candidate-only' }} |
| 1248 | runs-on: ubuntu-latest |
| 1249 | permissions: |
| 1250 | actions: read |
| 1251 | contents: read |
| 1252 | pages: write |
| 1253 | id-token: write |
| 1254 | concurrency: |
| 1255 | group: pages |
| 1256 | cancel-in-progress: true |
| 1257 | environment: |
| 1258 | name: github-pages |
| 1259 | url: ${{ steps.deployment.outputs.page_url }} |
| 1260 | env: |
| 1261 | HUGO_VERSION: 0.161.1 |
| 1262 | |
| 1263 | steps: |
| 1264 | - name: Check out repository |
| 1265 | uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 |
| 1266 | with: |
| 1267 | submodules: recursive |
| 1268 | fetch-depth: 0 |
| 1269 | ref: ${{ github.event.repository.default_branch }} |
| 1270 | persist-credentials: false |
| 1271 | |
| 1272 | - name: Download raw crawl artifact |
| 1273 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1274 | with: |
| 1275 | name: raw-data |
| 1276 | path: data/raw/ |
| 1277 | |
| 1278 | - name: Download analyzed data artifact |
| 1279 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1280 | with: |
| 1281 | name: promoted-analyzed-data |
| 1282 | path: data/analyzed/ |
| 1283 | |
| 1284 | - name: Download generated content artifact |
| 1285 | uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1286 | with: |
| 1287 | name: generated-content |
| 1288 | path: content/ |
| 1289 | merge-multiple: true |
| 1290 | |
| 1291 | # Deploy builds from artifacts — no dependency on PR merges to main. |
| 1292 | - name: Configure GitHub Pages |
| 1293 | uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 |
| 1294 | |
| 1295 | - name: Install Hugo |
| 1296 | run: | |
| 1297 | set -euo pipefail |
| 1298 | RELEASE_URL="https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}" |
| 1299 | TARBALL="hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz" |
| 1300 | CHECKSUM_FILE="hugo_${HUGO_VERSION}_checksums.txt" |
| 1301 | curl --fail --silent --show-error --location --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors \ |
| 1302 | --output "${TARBALL}" "${RELEASE_URL}/${TARBALL}" |
| 1303 | curl --fail --silent --show-error --location --retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors \ |
| 1304 | --output "${CHECKSUM_FILE}" "${RELEASE_URL}/${CHECKSUM_FILE}" |
| 1305 | checksum_line="$(awk -v file="${TARBALL}" '$NF == file {print; found=1} END {if (!found) exit 1}' "${CHECKSUM_FILE}")" || { |
| 1306 | echo "Error: No checksum entry for ${TARBALL} found in ${CHECKSUM_FILE}" >&2 |
| 1307 | exit 1 |
| 1308 | } |
| 1309 | if ! printf '%s\n' "${checksum_line}" | sha256sum --check; then |
| 1310 | echo "Error: Checksum verification failed for ${TARBALL}" >&2 |
| 1311 | exit 1 |
| 1312 | fi |
| 1313 | rm "${CHECKSUM_FILE}" |
| 1314 | mkdir -p "${HOME}/.local/hugo" |
| 1315 | tar -C "${HOME}/.local/hugo" -xf "${TARBALL}" |
| 1316 | rm "${TARBALL}" |
| 1317 | echo "${HOME}/.local/hugo" >> "${GITHUB_PATH}" |
| 1318 | export PATH="${HOME}/.local/hugo:${PATH}" |
| 1319 | hugo version |
| 1320 | |
| 1321 | - name: Build site |
| 1322 | run: hugo --minify |
| 1323 | |
| 1324 | - name: Build search index |
| 1325 | run: npx pagefind --site public/ |
| 1326 | |
| 1327 | - name: Upload Pages artifact |
| 1328 | uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 |
| 1329 | with: |
| 1330 | path: ./public |
| 1331 | |
| 1332 | - name: Deploy to GitHub Pages |
| 1333 | id: deployment |
| 1334 | uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 |
| 1335 | |
| 1336 | # NOTE: The Podcaster handoff intentionally lives in sync-publish-to-main.yml, |
| 1337 | # which fires only AFTER the weekly article is merged into main. Triggering it |
| 1338 | # here (off deploy) created a race where the podcaster could start before the |
| 1339 | # article was merged, yielding stub episodes (e.g. W27). Do not re-add a |
| 1340 | # deploy-coupled handoff job. |
| 1341 | |
| 1342 | notify: |
| 1343 | if: ${{ (github.event_name == 'schedule' || github.event.inputs.publish_release == 'true') && needs.analyze.outputs.run_mode != 'dry-run' && needs.analyze.outputs.run_mode != 'candidate-only' }} |
| 1344 | needs: [analyze, generate, deploy] |
| 1345 | runs-on: ubuntu-latest |
| 1346 | permissions: |
| 1347 | contents: write |
| 1348 | discussions: write |
| 1349 | |
| 1350 | steps: |
| 1351 | - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 |
| 1352 | with: |
| 1353 | persist-credentials: false |
| 1354 | |
| 1355 | - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1356 | with: |
| 1357 | name: promoted-analyzed-data |
| 1358 | path: data/analyzed/ |
| 1359 | |
| 1360 | - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 |
| 1361 | with: |
| 1362 | name: analysis-candidate |
| 1363 | path: data/candidates/ |
| 1364 | |
| 1365 | - name: Create GitHub Release |
| 1366 | env: |
| 1367 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 1368 | SUMMARY_FILE: ${{ needs.analyze.outputs.summary_file }} |
| 1369 | MANIFEST_FILE: ${{ needs.analyze.outputs.publish_manifest_file }} |
| 1370 | run: | |
| 1371 | python3 scripts/publish_manifest.py assert-eligible --manifest "$MANIFEST_FILE" |
| 1372 | WEEK=$(basename "$SUMMARY_FILE" | sed 's/-summary.md//') |
| 1373 | TAG="week-${WEEK}" |
| 1374 | TITLE="Week ${WEEK} — Tech Trends Summary" |
| 1375 | |
| 1376 | if gh release view "$TAG" >/dev/null 2>&1; then |
| 1377 | echo "::notice::Release $TAG already exists; updating it instead of failing." |
| 1378 | gh release edit "$TAG" --title "$TITLE" --notes-file "$SUMMARY_FILE" --latest |
| 1379 | else |
| 1380 | gh release create "$TAG" --title "$TITLE" --notes-file "$SUMMARY_FILE" --latest |
| 1381 | fi |
| 1382 | |
| 1383 | - name: Post to Discussions |
| 1384 | env: |
| 1385 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 1386 | REPO_OWNER: ${{ github.repository_owner }} |
| 1387 | REPO_NAME: ${{ github.event.repository.name }} |
| 1388 | GH_REPOSITORY: ${{ github.repository }} |
| 1389 | run: | |
| 1390 | set -e |
| 1391 | |
| 1392 | SUMMARY=$(ls -t data/analyzed/*-summary.md | head -1) |
| 1393 | WEEK=$(basename "$SUMMARY" | sed 's/-summary.md//') |
| 1394 | |
| 1395 | # Create discussion via GraphQL (Announcements category) |
| 1396 | REPO_ID=$(gh api "repos/$GH_REPOSITORY" --jq '.node_id') |
| 1397 | if [ -z "$REPO_ID" ]; then |
| 1398 | echo "::error::Failed to get repository node_id" |
| 1399 | exit 1 |
| 1400 | fi |
| 1401 | |
| 1402 | CAT_ID=$(gh api graphql -f query='{ repository(owner:"'"$REPO_OWNER"'", name:"'"$REPO_NAME"'") { discussionCategories(first:10) { nodes { id name } } } }' --jq '.data.repository.discussionCategories.nodes[] | select(.name=="Announcements") | .id' 2>/dev/null || echo "") |
| 1403 | if [ -z "$CAT_ID" ]; then |
| 1404 | echo "::warning::Announcements discussion category not found; skipping discussion creation" |
| 1405 | exit 0 |
| 1406 | fi |
| 1407 | |
| 1408 | # Safely encode body as JSON |
| 1409 | BODY=$(jq -Rs . < "$SUMMARY") |
| 1410 | |
| 1411 | gh api graphql -f query="mutation { createDiscussion(input: {repositoryId: \"$REPO_ID\", categoryId: \"$CAT_ID\", title: \"Week $WEEK — Tech Trends Summary\", body: $BODY}) { discussion { url } } }" |
| 1412 | |
| 1413 | - name: Post to webhook |
| 1414 | if: env.WEBHOOK_URL != '' |
| 1415 | env: |
| 1416 | WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} |
| 1417 | run: | |
| 1418 | SUMMARY=$(ls -t data/analyzed/*-summary.md | head -1) |
| 1419 | WEEK=$(basename "$SUMMARY" | sed 's/-summary.md//') |
| 1420 | SITE_URL="https://jmservera.github.io/SquadScope/weekly/$(echo "$WEEK" | tr '-' '/' | sed 's/W/w/')/" |
| 1421 | |
| 1422 | # Build JSON payload with jq to prevent injection via WEEK or SITE_URL |
| 1423 | PAYLOAD=$(jq -n \ |
| 1424 | --arg content "📊 **SquadScope Week ${WEEK}** — New tech trends summary published!\n${SITE_URL}" \ |
| 1425 | --arg username "SquadScope" \ |
| 1426 | '{content: $content, username: $username}') |
| 1427 | |
| 1428 | curl -s -X POST "$WEBHOOK_URL" \ |
| 1429 | -H "Content-Type: application/json" \ |
| 1430 | -d "$PAYLOAD" || echo "Webhook post failed (non-critical)" |
| 1431 | |
| 1432 | notify-failure: |
| 1433 | needs: [crawl, analyze, generate, deploy, notify] |
| 1434 | if: ${{ always() && contains(needs.*.result, 'failure') }} |
| 1435 | runs-on: ubuntu-latest |
| 1436 | permissions: |
| 1437 | actions: read |
| 1438 | issues: write |
| 1439 | |
| 1440 | steps: |
| 1441 | - name: Create or update failure issue |
| 1442 | env: |
| 1443 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 1444 | GH_REPO: ${{ github.repository }} |
| 1445 | SERVER_URL: ${{ github.server_url }} |
| 1446 | RUN_ID: ${{ github.run_id }} |
| 1447 | run: | |
| 1448 | set -euo pipefail |
| 1449 | RUN_URL="${SERVER_URL}/${GH_REPO}/actions/runs/${RUN_ID}" |
| 1450 | FAILED_JOBS=$(gh run view "$RUN_ID" --json jobs --jq '[.jobs[] | select(.conclusion=="failure") | .name] | join(", ")') |
| 1451 | if [ -z "$FAILED_JOBS" ]; then |
| 1452 | FAILED_JOBS="unknown" |
| 1453 | fi |
| 1454 | |
| 1455 | EXISTING=$(gh issue list --state open --search 'in:title "Crawl and publish pipeline failed"' --json number --jq length) |
| 1456 | if [ "$EXISTING" -gt 0 ]; then |
| 1457 | ISSUE_NUM=$(gh issue list --state open --search 'in:title "Crawl and publish pipeline failed"' --json number --jq '.[0].number') |
| 1458 | COMMENT_BODY=$(printf '%s\n\n%s' \ |
| 1459 | "Pipeline failed again: ${RUN_URL}" \ |
| 1460 | "Failed jobs: ${FAILED_JOBS}") |
| 1461 | gh issue comment "$ISSUE_NUM" --body "$COMMENT_BODY" |
| 1462 | echo "Updated existing issue #$ISSUE_NUM" |
| 1463 | else |
| 1464 | BODY=$(printf '%s\n\n%s\n%s\n\n%s\n%s\n%s' \ |
| 1465 | "The Crawl and publish weekly data workflow failed." \ |
| 1466 | "**Run:** ${RUN_URL}" \ |
| 1467 | "**Failed jobs:** ${FAILED_JOBS}" \ |
| 1468 | "Please triage:" \ |
| 1469 | "- If transient (network/rate limit), close with context" \ |
| 1470 | "- If real bug, assign to the right squad member") |
| 1471 | gh issue create \ |
| 1472 | --title "🔴 Crawl and publish pipeline failed (run ${RUN_ID})" \ |
| 1473 | --label bug \ |
| 1474 | --label squad \ |
| 1475 | --body "$BODY" |
| 1476 | fi |