| 1 | import io |
| 2 | import json |
| 3 | import tempfile |
| 4 | import unittest |
| 5 | from argparse import Namespace |
| 6 | from datetime import UTC, datetime |
| 7 | from pathlib import Path |
| 8 | from unittest import mock |
| 9 | |
| 10 | import yaml |
| 11 | |
| 12 | import scripts.analysis_gate as analysis_gate |
| 13 | import scripts.analyze_fallback as analyze_fallback |
| 14 | import scripts.crawl as crawl |
| 15 | import scripts.generate_content as generate_content |
| 16 | |
| 17 | |
| 18 | def _uses_action(step: dict, action: str) -> bool: |
| 19 | """Return True if a step uses ``action``, ignoring the version/SHA ref. |
| 20 | |
| 21 | Tolerates SHA-pinned references such as |
| 22 | ``actions/download-artifact@<40-hex-sha> # v4`` by comparing only the |
| 23 | ``owner/repo`` portion before the ``@``. |
| 24 | """ |
| 25 | uses = step.get("uses") |
| 26 | if not isinstance(uses, str): |
| 27 | return False |
| 28 | return uses.split("@", 1)[0] == action |
| 29 | |
| 30 | |
| 31 | class _FakeHTTPResponse(io.BytesIO): |
| 32 | def __enter__(self): |
| 33 | return self |
| 34 | |
| 35 | def __exit__(self, exc_type, exc, tb): |
| 36 | self.close() |
| 37 | return False |
| 38 | |
| 39 | |
| 40 | FIXED_RUN_DATETIME = "2026-05-18T08:00:00Z" |
| 41 | FIXED_RUN_TIME = datetime(2026, 5, 18, 8, 0, 0, tzinfo=UTC) |
| 42 | |
| 43 | |
| 44 | def make_api_repo(full_name: str, *, stars: int, created_at: str, topics: list[str]) -> dict: |
| 45 | owner, name = full_name.split("/", 1) |
| 46 | return { |
| 47 | "name": name, |
| 48 | "full_name": full_name, |
| 49 | "description": f"{name} helps teams ship reliable automation.", |
| 50 | "language": "Python", |
| 51 | "stargazers_count": stars, |
| 52 | "forks_count": max(1, stars // 10), |
| 53 | "created_at": created_at, |
| 54 | "topics": topics, |
| 55 | "license": {"spdx_id": "MIT"}, |
| 56 | "html_url": f"https://github.com/{full_name}", |
| 57 | "owner": {"login": owner}, |
| 58 | "fork": False, |
| 59 | "is_template": False, |
| 60 | } |
| 61 | |
| 62 | |
| 63 | def make_raw_payload() -> dict: |
| 64 | return { |
| 65 | "week": "2026-W21", |
| 66 | "crawled_at": FIXED_RUN_DATETIME, |
| 67 | "new_repos": [ |
| 68 | { |
| 69 | "name": "signal-kit", |
| 70 | "owner": "octo", |
| 71 | "full_name": "octo/signal-kit", |
| 72 | "description": "Signal extraction for release teams.", |
| 73 | "language": "Python", |
| 74 | "stars": 120, |
| 75 | "forks": 12, |
| 76 | "created_at": "2026-05-12T09:00:00Z", |
| 77 | "topics": ["ai", "automation", "developer-tooling"], |
| 78 | "license": "MIT", |
| 79 | "url": "https://github.com/octo/signal-kit", |
| 80 | } |
| 81 | ], |
| 82 | "trending_repos": [ |
| 83 | { |
| 84 | "name": "momentum-watch", |
| 85 | "owner": "octo", |
| 86 | "full_name": "octo/momentum-watch", |
| 87 | "description": "Observability for weekly launches.", |
| 88 | "language": "Go", |
| 89 | "stars": 180, |
| 90 | "forks": 18, |
| 91 | "created_at": "2026-05-10T12:00:00Z", |
| 92 | "topics": ["observability", "analytics", "platform"], |
| 93 | "license": "Apache-2.0", |
| 94 | "url": "https://github.com/octo/momentum-watch", |
| 95 | "stars_gained": 35, |
| 96 | } |
| 97 | ], |
| 98 | "signals": { |
| 99 | "top_topics": [ |
| 100 | {"topic": "automation", "count": 2}, |
| 101 | {"topic": "observability", "count": 1}, |
| 102 | ] |
| 103 | }, |
| 104 | "metadata": { |
| 105 | "api_calls_used": 2, |
| 106 | "cache_hits": 1, |
| 107 | "stale_cache_hits": 0, |
| 108 | "rate_limit_limit": 5000, |
| 109 | "rate_limit_remaining": 4990, |
| 110 | "rate_limit_reset": 1747567200, |
| 111 | "rate_limit_resource": "search", |
| 112 | "partial_failures": [], |
| 113 | "snapshot_path": "data/snapshots/2026-W21-stars.json", |
| 114 | }, |
| 115 | } |
| 116 | |
| 117 | |
| 118 | def make_analysis_markdown() -> str: |
| 119 | return f"""--- |
| 120 | title: "Reliable Automation Gains Ground" |
| 121 | date: {FIXED_RUN_DATETIME} |
| 122 | week: "2026-W21" |
| 123 | year: 2026 |
| 124 | tags: [ai, automation, developer-tooling] |
| 125 | categories: [weekly] |
| 126 | repos_featured: 2 |
| 127 | stars_tracked: 300 |
| 128 | top_repo: "octo/signal-kit" |
| 129 | quality_score: 86 |
| 130 | summary: "Reliable automation and observability projects set the tone for the week." |
| 131 | --- |
| 132 | |
| 133 | ## This Week's Trends |
| 134 | |
| 135 | **Operational Automation**: Teams are investing in tools that reduce coordination overhead and improve release confidence. [octo/signal-kit](https://github.com/octo/signal-kit) exemplifies this — it solves release coordination without pretending to be a full platform rewrite. The project packages practical automation, readable defaults, and evidence of disciplined engineering. |
| 136 | |
| 137 | **Observability as Infrastructure**: [octo/momentum-watch](https://github.com/octo/momentum-watch) captured attention because the work is grounded in run health and measurement rather than novelty claims. The trend matters because more teams are prioritizing incident feedback loops and durable visibility into developer workflows. |
| 138 | |
| 139 | ## Where Industry Meets Code |
| 140 | |
| 141 | Developer activity aligned with broader industry interest in automation and observability tooling this week. Both [octo/signal-kit](https://github.com/octo/signal-kit) and [octo/momentum-watch](https://github.com/octo/momentum-watch) represent categories where press coverage and developer investment point in the same direction. The more interesting divergence is what the press is not covering: the quiet growth of practical pipeline tooling that makes releases safer without requiring major architectural changes. This type of grounded infrastructure work rarely earns headlines, but this week's developer activity suggests it is where real adoption is happening. |
| 142 | |
| 143 | ## Signal & Noise |
| 144 | |
| 145 | The durable signal this week is a return to automation that lowers toil and gives teams more confidence in repeatable delivery. [octo/signal-kit](https://github.com/octo/signal-kit) and [octo/momentum-watch](https://github.com/octo/momentum-watch) both point toward software that reduces coordination overhead, improves trust in pipelines, and respects how operators actually work. That pattern is more convincing than broad claims about agents replacing engineering judgment. |
| 146 | |
| 147 | The noise is the usual rush of products that market autonomy without proving fit, maintenance discipline, or measurable outcomes. This week was healthier than most, but the broader ecosystem still produces wrappers that borrow the language of automation while skipping the hard parts of observability, testing, and operational ownership. |
| 148 | |
| 149 | ## Blind Spots |
| 150 | |
| 151 | The biggest blind spot is stronger investment in security review, test ergonomics, and smaller-team operations tooling that can be adopted without a platform migration. The ecosystem is getting better at coordination, but it still underserves practical defensive tooling and deployment confidence for teams that need reliability before they need spectacle. Neither press nor developer communities are giving this the attention it deserves. |
| 152 | |
| 153 | ## The Week Ahead |
| 154 | |
| 155 | Practical automation won attention on merit this week. If this pattern holds, the next wave of winners will be tools that save teams time, expose real operating signals, and make release quality easier to trust. Watch for observability and pipeline safety tooling to continue gaining ground. |
| 156 | |
| 157 | ## Key References |
| 158 | |
| 159 | ### Notable Projects |
| 160 | |
| 161 | - [octo/signal-kit](https://github.com/octo/signal-kit) — release coordination automation with practical defaults and disciplined engineering. |
| 162 | - [octo/momentum-watch](https://github.com/octo/momentum-watch) — observability tooling grounded in run health rather than vanity metrics. |
| 163 | |
| 164 | ### Press & Industry |
| 165 | |
| 166 | No press data was provided this week. |
| 167 | """ |
| 168 | |
| 169 | |
| 170 | class WorkflowConfigTests(unittest.TestCase): |
| 171 | def test_hugo_install_steps_use_release_urls_and_resilient_retries(self) -> None: |
| 172 | expected_retry_flags = "--retry 10 --retry-delay 5 --retry-max-time 300 --retry-all-errors" |
| 173 | |
| 174 | for workflow_file in ( |
| 175 | ".github/workflows/deploy-site.yml", |
| 176 | ".github/workflows/crawl-and-publish.yml", |
| 177 | ): |
| 178 | workflow_path = Path(workflow_file) |
| 179 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 180 | |
| 181 | install_step = next( |
| 182 | ( |
| 183 | step |
| 184 | for job in workflow["jobs"].values() |
| 185 | for step in job.get("steps", []) |
| 186 | if step.get("name") == "Install Hugo" |
| 187 | ), |
| 188 | None, |
| 189 | ) |
| 190 | self.assertIsNotNone(install_step, f"Install Hugo step not found in {workflow_file}") |
| 191 | install_run = install_step["run"] |
| 192 | self.assertIn( |
| 193 | 'RELEASE_URL="https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}"', |
| 194 | install_run, |
| 195 | ) |
| 196 | self.assertIn('TARBALL="hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz"', install_run) |
| 197 | self.assertIn('CHECKSUM_FILE="hugo_${HUGO_VERSION}_checksums.txt"', install_run) |
| 198 | self.assertEqual(install_run.count(expected_retry_flags), 2) |
| 199 | |
| 200 | def test_crawl_workflow_persists_run_counter(self) -> None: |
| 201 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 202 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 203 | |
| 204 | crawl_job = workflow["jobs"]["crawl"] |
| 205 | commit_step = None |
| 206 | for step in crawl_job["steps"]: |
| 207 | if step.get("name") == "Commit crawl data to data branch": |
| 208 | commit_step = step |
| 209 | break |
| 210 | |
| 211 | self.assertIsNotNone(commit_step, "Commit crawl data to data branch step not found") |
| 212 | run_script = commit_step["run"] |
| 213 | self.assertIn("COUNTER=$(cat .squad/run-counter.txt", run_script) |
| 214 | self.assertIn("COUNTER=$((COUNTER + 1))", run_script) |
| 215 | self.assertIn(".squad/run-counter.txt", run_script) |
| 216 | self.assertIn("git add data/raw/ data/snapshots/ data/raw-store/", run_script) |
| 217 | self.assertIn("git add .squad/run-counter.txt", run_script) |
| 218 | |
| 219 | def test_external_news_workflow_passes_deterministic_until(self) -> None: |
| 220 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 221 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 222 | |
| 223 | crawl_job = workflow["jobs"]["crawl"] |
| 224 | external_news_step = next( |
| 225 | ( |
| 226 | step |
| 227 | for step in crawl_job["steps"] |
| 228 | if step.get("name") == "Crawl external news RSS feeds" |
| 229 | ), |
| 230 | None, |
| 231 | ) |
| 232 | |
| 233 | self.assertIsNotNone(external_news_step, "External news crawl step not found") |
| 234 | run_script = external_news_step["run"] |
| 235 | self.assertIn("SINCE=$(date -u -d '7 days ago' +%Y-%m-%d)", run_script) |
| 236 | self.assertIn("UNTIL=$(date -u +%Y-%m-%d)", run_script) |
| 237 | self.assertIn('--until "$UNTIL"', run_script) |
| 238 | |
| 239 | def test_crawl_workflow_defines_analyze_job(self) -> None: |
| 240 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 241 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 242 | |
| 243 | # Reskill jobs have been removed (analysis uses plain copilot-cli) |
| 244 | self.assertNotIn("reskill-check", workflow["jobs"]) |
| 245 | self.assertNotIn("reskill", workflow["jobs"]) |
| 246 | |
| 247 | analyze = workflow["jobs"]["analyze"] |
| 248 | preflight_step = next( |
| 249 | ( |
| 250 | s |
| 251 | for s in analyze["steps"] |
| 252 | if s.get("name") == "Render and preflight analysis prompt" |
| 253 | ), |
| 254 | None, |
| 255 | ) |
| 256 | self.assertIsNotNone(preflight_step) |
| 257 | preflight_run = preflight_step["run"] |
| 258 | self.assertIn("--prompt-token-budget", preflight_run) |
| 259 | self.assertIn("--preflight-report-json", preflight_run) |
| 260 | self.assertIn("--preflight-report-md", preflight_run) |
| 261 | self.assertIn('--print-prompt > "$PROMPT_FILE"', preflight_run) |
| 262 | self.assertIn('--context-files "$PROMPT_FILE"', preflight_run) |
| 263 | self.assertIn("promotion_policy=", preflight_run) |
| 264 | self.assertIn("staged/candidate-only", preflight_run) |
| 265 | |
| 266 | run_analysis_step = next( |
| 267 | (s for s in analyze["steps"] if s.get("name") == "Run analysis"), None |
| 268 | ) |
| 269 | self.assertIsNotNone(run_analysis_step) |
| 270 | run_analysis = run_analysis_step["run"] |
| 271 | self.assertIn("python3 scripts/track_token_usage.py", run_analysis) |
| 272 | self.assertIn('ANALYSIS_MODEL="copilot-default"', run_analysis) |
| 273 | self.assertNotIn("--model claude-sonnet-4", run_analysis) |
| 274 | self.assertIn("mkdir -p data/metrics", run_analysis) |
| 275 | self.assertIn("run_quality_gate()", run_analysis) |
| 276 | self.assertIn("python3 scripts/copilot_failure.py", run_analysis) |
| 277 | self.assertIn("--create-token-issue", run_analysis) |
| 278 | self.assertIn('FINAL_FAILURE_CLASS=""', run_analysis) |
| 279 | self.assertIn("--agent weekly-analysis", run_analysis) |
| 280 | self.assertIn( |
| 281 | "Read the file at ${PROMPT_FILE}. Write the complete weekly analysis markdown to ${OUTPUT_FILE}.", |
| 282 | run_analysis, |
| 283 | ) |
| 284 | self.assertIn('if ! test -s "$OUTPUT_FILE"; then', run_analysis) |
| 285 | self.assertIn('FINAL_FAILURE_CLASS="writer_contract_failure"', run_analysis) |
| 286 | self.assertNotIn("--allow-tool=glob", run_analysis) |
| 287 | self.assertNotIn("--allow-tool=grep", run_analysis) |
| 288 | self.assertIn( |
| 289 | 'if [ "$FAILURE_CLASS" = "copilot_token_failure" ] || [ "$FAILURE_CLASS" = "copilot_inaccessible" ]; then', |
| 290 | run_analysis, |
| 291 | ) |
| 292 | self.assertIn("failing without no-AI fallback", run_analysis) |
| 293 | self.assertIn( |
| 294 | 'echo "copilot is not available: command not found" > "$COPILOT_LOG"', run_analysis |
| 295 | ) |
| 296 | self.assertIn("--exit-code 127", run_analysis) |
| 297 | self.assertIn("No publishable Copilot summary was produced", run_analysis) |
| 298 | self.assertIn("current published article can be preserved", run_analysis) |
| 299 | self.assertIn("python3 scripts/analyze_fallback.py", run_analysis) |
| 300 | self.assertIn('--press-context "$PRESS_FILE"', run_analysis) |
| 301 | self.assertIn("--no-ai", run_analysis) |
| 302 | self.assertIn('ANALYSIS_SOURCE="no-ai"', run_analysis) |
| 303 | self.assertNotIn('ANALYSIS_SOURCE="github-models"', run_analysis) |
| 304 | self.assertNotIn("falling back to GitHub Models API", run_analysis) |
| 305 | |
| 306 | manifest_step = next( |
| 307 | (s for s in analyze["steps"] if s.get("name") == "Emit publish eligibility manifest"), |
| 308 | None, |
| 309 | ) |
| 310 | self.assertIsNotNone(manifest_step) |
| 311 | manifest_run = manifest_step["run"] |
| 312 | self.assertEqual( |
| 313 | manifest_step["env"]["PREFLIGHT_REPORT"], |
| 314 | "${{ steps.prompt-preflight.outputs.preflight_report_json }}", |
| 315 | ) |
| 316 | self.assertIn('--preflight-report "$PREFLIGHT_REPORT"', manifest_run) |
| 317 | |
| 318 | def test_generate_workflow_runs_rollups_and_commits_all_content(self) -> None: |
| 319 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 320 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 321 | |
| 322 | deploy_job = workflow["jobs"]["deploy"] |
| 323 | build_site_step = next( |
| 324 | (s for s in deploy_job["steps"] if s.get("name") == "Build site"), None |
| 325 | ) |
| 326 | self.assertIsNotNone(build_site_step) |
| 327 | self.assertEqual(build_site_step["run"], "hugo --minify") |
| 328 | |
| 329 | pagefind_step = next( |
| 330 | (s for s in deploy_job["steps"] if s.get("name") == "Build search index"), None |
| 331 | ) |
| 332 | self.assertIsNotNone(pagefind_step) |
| 333 | self.assertEqual(pagefind_step["run"], "npx pagefind --site public/") |
| 334 | |
| 335 | generate_job = workflow["jobs"]["generate"] |
| 336 | generate_rollups_step = next( |
| 337 | (s for s in generate_job["steps"] if s.get("name") == "Generate rollups"), None |
| 338 | ) |
| 339 | self.assertIsNotNone(generate_rollups_step) |
| 340 | self.assertEqual(generate_rollups_step["run"], "python3 scripts/generate_rollups.py") |
| 341 | |
| 342 | commit_step = next( |
| 343 | ( |
| 344 | s |
| 345 | for s in generate_job["steps"] |
| 346 | if s.get("name") == "Commit generated content to data branch" |
| 347 | ), |
| 348 | None, |
| 349 | ) |
| 350 | self.assertIsNotNone(commit_step) |
| 351 | commit_run = commit_step["run"] |
| 352 | self.assertIn("content/weekly", commit_run) |
| 353 | self.assertIn("content/monthly", commit_run) |
| 354 | self.assertIn("content/yearly", commit_run) |
| 355 | self.assertIn("content/weekly/", commit_run) |
| 356 | self.assertIn("content/monthly/", commit_run) |
| 357 | self.assertIn("content/yearly/", commit_run) |
| 358 | self.assertIn("GITHUB_WORKSPACE", commit_run) |
| 359 | self.assertIn('case "$PAGE_PATH" in', commit_run) |
| 360 | self.assertIn("Expected PAGE_PATH under content/weekly/", commit_run) |
| 361 | |
| 362 | upload_step = next( |
| 363 | ( |
| 364 | s |
| 365 | for s in generate_job["steps"] |
| 366 | if s.get("name") == "Upload generated content artifact" |
| 367 | ), |
| 368 | None, |
| 369 | ) |
| 370 | self.assertIsNotNone(upload_step) |
| 371 | self.assertIn("content/monthly/", upload_step["with"]["path"]) |
| 372 | self.assertIn("content/yearly/", upload_step["with"]["path"]) |
| 373 | promoted_upload = next( |
| 374 | ( |
| 375 | s |
| 376 | for s in generate_job["steps"] |
| 377 | if s.get("name") == "Upload promoted analyzed artifact" |
| 378 | ), |
| 379 | None, |
| 380 | ) |
| 381 | self.assertIsNotNone(promoted_upload) |
| 382 | self.assertEqual(promoted_upload["with"]["name"], "promoted-analyzed-data") |
| 383 | |
| 384 | def test_sync_publish_to_main_excludes_squad_state_and_regenerates_rollups(self) -> None: |
| 385 | workflow_path = Path(".github/workflows/sync-publish-to-main.yml") |
| 386 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 387 | |
| 388 | sync_job = workflow["jobs"]["sync"] |
| 389 | sync_step = next( |
| 390 | (s for s in sync_job["steps"] if s.get("name") == "Sync data from publish"), None |
| 391 | ) |
| 392 | self.assertIsNotNone(sync_step) |
| 393 | |
| 394 | sync_run = sync_step["run"] |
| 395 | for generated_path in ( |
| 396 | "data/raw/", |
| 397 | "data/analyzed/", |
| 398 | "data/metrics/", |
| 399 | "content/weekly/", |
| 400 | "content/monthly/", |
| 401 | "content/yearly/", |
| 402 | ): |
| 403 | self.assertIn(generated_path, sync_run) |
| 404 | |
| 405 | self.assertIn("python3 scripts/generate_rollups.py", sync_run) |
| 406 | self.assertLess( |
| 407 | sync_run.index("python3 scripts/generate_rollups.py"), sync_run.index("git add -A") |
| 408 | ) |
| 409 | self.assertIn("Refusing to sync .squad state from publish to main.", sync_run) |
| 410 | self.assertLess(sync_run.index("Refusing to sync .squad"), sync_run.index("git commit -m")) |
| 411 | self.assertIn("**Explicitly NOT synced:**", sync_run) |
| 412 | self.assertIn(".squad/**", sync_run) |
| 413 | self.assertNotIn("git checkout origin/publish -- .squad", sync_run) |
| 414 | self.assertNotIn("git ls-tree -r --name-only origin/publish -- .squad", sync_run) |
| 415 | self.assertNotIn(".squad/decisions.md", sync_run) |
| 416 | self.assertNotIn(".squad/agents/*/history.md", sync_run) |
| 417 | self.assertNotIn("squad learnings", sync_run.lower()) |
| 418 | |
| 419 | def test_notify_workflow_posts_optional_webhook(self) -> None: |
| 420 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 421 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 422 | |
| 423 | notify_job = workflow["jobs"]["notify"] |
| 424 | self.assertEqual(notify_job["needs"], ["analyze", "generate", "deploy"]) |
| 425 | analyzed_download = next( |
| 426 | ( |
| 427 | s |
| 428 | for s in notify_job["steps"] |
| 429 | if _uses_action(s, "actions/download-artifact") |
| 430 | and s.get("with", {}).get("path") == "data/analyzed/" |
| 431 | ), |
| 432 | None, |
| 433 | ) |
| 434 | self.assertIsNotNone(analyzed_download) |
| 435 | self.assertEqual(analyzed_download["with"]["name"], "promoted-analyzed-data") |
| 436 | |
| 437 | webhook_step = next( |
| 438 | (s for s in notify_job["steps"] if s.get("name") == "Post to webhook"), None |
| 439 | ) |
| 440 | self.assertIsNotNone(webhook_step) |
| 441 | self.assertEqual(webhook_step["if"], "env.WEBHOOK_URL != ''") |
| 442 | self.assertEqual(webhook_step["env"]["WEBHOOK_URL"], "${{ secrets.WEBHOOK_URL }}") |
| 443 | |
| 444 | release_step = next( |
| 445 | (s for s in notify_job["steps"] if s.get("name") == "Create GitHub Release"), None |
| 446 | ) |
| 447 | self.assertIsNotNone(release_step) |
| 448 | self.assertEqual( |
| 449 | release_step["env"]["SUMMARY_FILE"], "${{ needs.analyze.outputs.summary_file }}" |
| 450 | ) |
| 451 | release_run = release_step["run"] |
| 452 | self.assertIn('gh release view "$TAG"', release_run) |
| 453 | self.assertIn('gh release edit "$TAG"', release_run) |
| 454 | self.assertIn('gh release create "$TAG"', release_run) |
| 455 | |
| 456 | webhook_run = webhook_step["run"] |
| 457 | self.assertIn('curl -s -X POST "$WEBHOOK_URL"', webhook_run) |
| 458 | self.assertIn("https://jmservera.github.io/SquadScope/weekly/", webhook_run) |
| 459 | # JSON is now built with jq to prevent injection — check for jq invocation |
| 460 | self.assertIn("jq -n", webhook_run) |
| 461 | self.assertIn("📊 **SquadScope Week", webhook_run) |
| 462 | self.assertIn("Webhook post failed (non-critical)", webhook_run) |
| 463 | |
| 464 | def test_podcaster_handoff_triggers_post_merge_from_sync_not_crawl(self) -> None: |
| 465 | # Handoff must fire only AFTER the weekly article is merged to main, so it |
| 466 | # lives in sync-publish-to-main (post-merge), not in crawl-and-publish |
| 467 | # (which deploys from artifacts before the merge — the W27 stub race). |
| 468 | crawl = yaml.safe_load( |
| 469 | Path(".github/workflows/crawl-and-publish.yml").read_text(encoding="utf-8") |
| 470 | ) |
| 471 | self.assertNotIn("podcaster-handoff", crawl["jobs"]) |
| 472 | deploy_job = crawl["jobs"]["deploy"] |
| 473 | self.assertEqual(deploy_job["needs"], ["crawl", "analyze", "generate"]) |
| 474 | |
| 475 | sync = yaml.safe_load( |
| 476 | Path(".github/workflows/sync-publish-to-main.yml").read_text(encoding="utf-8") |
| 477 | ) |
| 478 | steps = sync["jobs"]["sync"]["steps"] |
| 479 | trigger = next((s for s in steps if s.get("name") == "Trigger Podcaster after merge"), None) |
| 480 | self.assertIsNotNone(trigger) |
| 481 | self.assertEqual(trigger["if"], "${{ steps.sync.outputs.merged == 'true' }}") |
| 482 | self.assertEqual(trigger["env"]["PODCASTER_ENDPOINT"], "${{ vars.PODCASTER_ENDPOINT }}") |
| 483 | self.assertEqual(trigger["env"]["PODCASTER_API_KEY"], "${{ secrets.PODCASTER_API_KEY }}") |
| 484 | run_script = trigger["run"] |
| 485 | self.assertIn("scripts/publish_manifest.py assert-eligible", run_script) |
| 486 | self.assertIn("scripts/podcaster_handoff.py", run_script) |
| 487 | self.assertIn("--require-merged", run_script) |
| 488 | self.assertNotIn("--force", run_script) |
| 489 | self.assertNotIn("echo $PODCASTER_API_KEY", run_script) |
| 490 | |
| 491 | def test_podcaster_smoke_workflow_exercises_real_weekly_payload_shape(self) -> None: |
| 492 | workflow_path = Path(".github/workflows/podcaster-handoff-smoke.yml") |
| 493 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 494 | |
| 495 | inputs = workflow[True]["workflow_dispatch"]["inputs"] |
| 496 | self.assertIn("week", inputs) |
| 497 | self.assertIn("article_url", inputs) |
| 498 | self.assertIn("article_path", inputs) |
| 499 | self.assertIn("article_sha256", inputs) |
| 500 | self.assertEqual(inputs["article_sha256"]["default"], "") |
| 501 | |
| 502 | smoke_job = workflow["jobs"]["smoke"] |
| 503 | smoke_step = next( |
| 504 | (s for s in smoke_job["steps"] if s.get("name") == "Smoke test Podcaster dry run"), None |
| 505 | ) |
| 506 | self.assertIsNotNone(smoke_step) |
| 507 | run_script = smoke_step["run"] |
| 508 | self.assertIn('if [ ! -f "$ARTICLE_PATH" ]', run_script) |
| 509 | self.assertIn("hashlib.sha256(article_bytes).hexdigest()", run_script) |
| 510 | self.assertIn("article_sha256 must match ARTICLE_PATH contents when provided.", run_script) |
| 511 | self.assertIn( |
| 512 | 'raw_payload = {"week": week, "source": "github", "article_path": article_path}', |
| 513 | run_script, |
| 514 | ) |
| 515 | self.assertIn('"size_bytes": len(raw_bytes)', run_script) |
| 516 | self.assertIn('"sha256": article_sha', run_script) |
| 517 | self.assertIn('"source_artifacts": [', run_script) |
| 518 | self.assertIn('"same_day_reuse"', run_script) |
| 519 | self.assertIn("build_payload(", run_script) |
| 520 | self.assertIn('"podcast_config"', run_script) |
| 521 | self.assertIn('"script_directions"', run_script) |
| 522 | self.assertIn('"spotify_publish"', run_script) |
| 523 | self.assertIn('"article_content"', run_script) |
| 524 | self.assertIn("--manifest .podcaster-smoke/publish-manifest.json", run_script) |
| 525 | self.assertIn("--podcast-config config/podcast.json", run_script) |
| 526 | self.assertIn("--podcaster-dry-run", run_script) |
| 527 | |
| 528 | def test_publish_workflow_uses_candidate_manifest_before_promotion(self) -> None: |
| 529 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 530 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 531 | |
| 532 | analyze = workflow["jobs"]["analyze"] |
| 533 | self.assertEqual( |
| 534 | analyze["outputs"]["summary_file"], |
| 535 | "${{ steps.analysis-context.outputs.published_output_file }}", |
| 536 | ) |
| 537 | self.assertEqual( |
| 538 | analyze["outputs"]["candidate_summary_file"], |
| 539 | "${{ steps.analysis-context.outputs.candidate_output_file }}", |
| 540 | ) |
| 541 | self.assertEqual( |
| 542 | analyze["outputs"]["publish_manifest_file"], |
| 543 | "${{ steps.analysis-context.outputs.publish_manifest_file }}", |
| 544 | ) |
| 545 | |
| 546 | prepare_step = next( |
| 547 | (s for s in analyze["steps"] if s.get("name") == "Prepare analysis context"), None |
| 548 | ) |
| 549 | self.assertIsNotNone(prepare_step) |
| 550 | prepare_run = prepare_step["run"] |
| 551 | self.assertIn("data/candidates", prepare_run) |
| 552 | self.assertIn("candidate_output_file", prepare_run) |
| 553 | self.assertIn("publish_manifest_file", prepare_run) |
| 554 | self.assertIn("published_output_file=data/analyzed", prepare_run) |
| 555 | |
| 556 | manifest_step = next( |
| 557 | (s for s in analyze["steps"] if s.get("name") == "Emit publish eligibility manifest"), |
| 558 | None, |
| 559 | ) |
| 560 | self.assertIsNotNone(manifest_step) |
| 561 | manifest_run = manifest_step["run"] |
| 562 | self.assertIn("scripts/publish_manifest.py create", manifest_run) |
| 563 | self.assertIn("--analysis-source", manifest_run) |
| 564 | self.assertIn("--analysis-model", manifest_run) |
| 565 | self.assertIn('--validation-status "$VALIDATION_STATUS"', manifest_run) |
| 566 | self.assertIn("--run-mode", manifest_run) |
| 567 | self.assertIn("--source-refresh-policy", manifest_run) |
| 568 | self.assertIn('git checkout origin/publish -- "$PUBLISHED_SUMMARY"', manifest_run) |
| 569 | |
| 570 | assert_step = next( |
| 571 | ( |
| 572 | s |
| 573 | for s in analyze["steps"] |
| 574 | if s.get("name") == "Assert candidate is eligible for promotion" |
| 575 | ), |
| 576 | None, |
| 577 | ) |
| 578 | self.assertIsNotNone(assert_step) |
| 579 | self.assertIn("scripts/publish_manifest.py assert-eligible", assert_step["run"]) |
| 580 | |
| 581 | self.assertEqual( |
| 582 | analyze["outputs"]["publish_head_sha"], "${{ steps.publish-base.outputs.sha }}" |
| 583 | ) |
| 584 | commit_step = next( |
| 585 | ( |
| 586 | s |
| 587 | for s in analyze["steps"] |
| 588 | if s.get("name") == "Commit analysis and learnings to data branch" |
| 589 | ), |
| 590 | None, |
| 591 | ) |
| 592 | self.assertIsNone(commit_step) |
| 593 | |
| 594 | upload_candidate = next( |
| 595 | (s for s in analyze["steps"] if s.get("name") == "Upload analysis candidate"), None |
| 596 | ) |
| 597 | self.assertIsNotNone(upload_candidate) |
| 598 | self.assertEqual(upload_candidate["if"], "always()") |
| 599 | |
| 600 | generate = workflow["jobs"]["generate"] |
| 601 | generate_raw_download = next( |
| 602 | ( |
| 603 | s |
| 604 | for s in generate["steps"] |
| 605 | if s.get("name") == "Download raw crawl artifact" |
| 606 | and _uses_action(s, "actions/download-artifact") |
| 607 | and s.get("with", {}).get("name") == "raw-data" |
| 608 | and s.get("with", {}).get("path") == "data/raw/" |
| 609 | ), |
| 610 | None, |
| 611 | ) |
| 612 | self.assertIsNotNone(generate_raw_download) |
| 613 | |
| 614 | generate_step = next( |
| 615 | (s for s in generate["steps"] if s.get("name") == "Generate weekly content"), None |
| 616 | ) |
| 617 | self.assertIsNotNone(generate_step) |
| 618 | self.assertIn('assert-eligible --manifest "$MANIFEST_FILE"', generate_step["run"]) |
| 619 | self.assertIn("candidate_content_path", generate_step["run"]) |
| 620 | self.assertIn("scripts/promotion_guard.py --manifest", generate_step["run"]) |
| 621 | |
| 622 | content_commit_step = next( |
| 623 | ( |
| 624 | s |
| 625 | for s in generate["steps"] |
| 626 | if s.get("name") == "Commit generated content to data branch" |
| 627 | ), |
| 628 | None, |
| 629 | ) |
| 630 | self.assertIsNotNone(content_commit_step) |
| 631 | content_commit_run = content_commit_step["run"] |
| 632 | self.assertIn( |
| 633 | "Publish branch drifted between analyze and content promotion", content_commit_run |
| 634 | ) |
| 635 | self.assertIn("backup-existing", content_commit_run) |
| 636 | self.assertIn('--path "data/published/${WEEK}/promotion-manifest.json"', content_commit_run) |
| 637 | self.assertIn("promotion-guard-tool.py --manifest", content_commit_run) |
| 638 | self.assertIn("data/published/", content_commit_run) |
| 639 | self.assertIn("--force-with-lease", content_commit_run) |
| 640 | |
| 641 | def test_rerun_mode_inputs_and_guards_are_declared(self) -> None: |
| 642 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 643 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 644 | inputs = workflow[True]["workflow_dispatch"]["inputs"] |
| 645 | |
| 646 | self.assertEqual(inputs["run_mode"]["default"], "normal") |
| 647 | self.assertIn("restore", inputs["run_mode"]["options"]) |
| 648 | self.assertEqual(inputs["source_refresh_policy"]["default"], "reuse-same-day") |
| 649 | self.assertIn("force-refresh", inputs["source_refresh_policy"]["options"]) |
| 650 | |
| 651 | crawl_steps = workflow["jobs"]["crawl"]["steps"] |
| 652 | validate_step = next( |
| 653 | (s for s in crawl_steps if s.get("name") == "Validate rerun mode"), None |
| 654 | ) |
| 655 | self.assertIsNotNone(validate_step) |
| 656 | self.assertIn("scripts/rerun_modes.py", validate_step["run"]) |
| 657 | |
| 658 | run_crawler = next((s for s in crawl_steps if s.get("name") == "Run crawler"), None) |
| 659 | self.assertIn("--reuse-artifact", run_crawler["run"]) |
| 660 | self.assertIn("--source-refresh-policy", run_crawler["run"]) |
| 661 | |
| 662 | def test_notify_failure_job_creates_or_updates_issue(self) -> None: |
| 663 | workflow_path = Path(".github/workflows/crawl-and-publish.yml") |
| 664 | workflow = yaml.safe_load(workflow_path.read_text(encoding="utf-8")) |
| 665 | |
| 666 | notify_failure_job = workflow["jobs"]["notify-failure"] |
| 667 | self.assertEqual( |
| 668 | notify_failure_job["needs"], ["crawl", "analyze", "generate", "deploy", "notify"] |
| 669 | ) |
| 670 | self.assertEqual( |
| 671 | notify_failure_job["if"], "${{ always() && contains(needs.*.result, 'failure') }}" |
| 672 | ) |
| 673 | self.assertEqual(notify_failure_job["permissions"], {"actions": "read", "issues": "write"}) |
| 674 | |
| 675 | create_issue_step = next( |
| 676 | ( |
| 677 | s |
| 678 | for s in notify_failure_job["steps"] |
| 679 | if s.get("name") == "Create or update failure issue" |
| 680 | ), |
| 681 | None, |
| 682 | ) |
| 683 | self.assertIsNotNone(create_issue_step) |
| 684 | self.assertEqual(create_issue_step["env"]["GITHUB_TOKEN"], "${{ secrets.GITHUB_TOKEN }}") |
| 685 | create_issue_run = create_issue_step["run"] |
| 686 | self.assertIn('gh run view "$RUN_ID" --json jobs', create_issue_run) |
| 687 | self.assertEqual(create_issue_step["env"]["RUN_ID"], "${{ github.run_id }}") |
| 688 | self.assertIn("gh issue list --state open --search", create_issue_run) |
| 689 | self.assertIn('gh issue comment "$ISSUE_NUM"', create_issue_run) |
| 690 | self.assertIn("gh issue create", create_issue_run) |
| 691 | self.assertIn("Crawl and publish pipeline failed", create_issue_run) |
| 692 | |
| 693 | |
| 694 | class PipelineIntegrationTests(unittest.TestCase): |
| 695 | def test_crawl_script_produces_valid_json_output_schema(self) -> None: |
| 696 | tests_root = Path(__file__).resolve().parent |
| 697 | with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir: |
| 698 | base = Path(tmpdir) |
| 699 | output_path = base / "data" / "raw" / "2026-W21.json" |
| 700 | snapshot_dir = base / "data" / "snapshots" |
| 701 | snapshot_dir.mkdir(parents=True) |
| 702 | |
| 703 | new_repo = make_api_repo( |
| 704 | "octo/signal-kit", |
| 705 | stars=120, |
| 706 | created_at="2026-05-12T09:00:00Z", |
| 707 | topics=["ai", "automation", "developer-tooling"], |
| 708 | ) |
| 709 | trending_repo = make_api_repo( |
| 710 | "octo/momentum-watch", |
| 711 | stars=180, |
| 712 | created_at="2026-05-10T12:00:00Z", |
| 713 | topics=["observability", "analytics", "platform"], |
| 714 | ) |
| 715 | |
| 716 | class FakeClient: |
| 717 | def __init__(self, token: str, **kwargs) -> None: |
| 718 | self.token = token |
| 719 | self.api_calls_used = 2 |
| 720 | self.cache_hits = 1 |
| 721 | self.stale_cache_hits = 0 |
| 722 | self.rate_limit_limit = 5000 |
| 723 | self.rate_limit_remaining = 4990 |
| 724 | self.rate_limit_reset = 1747567200 |
| 725 | self.rate_limit_resource = "search" |
| 726 | self.errors = [] |
| 727 | |
| 728 | def search_repositories(self, query: str, *, max_results: int = 1000): |
| 729 | if query.startswith("created:"): |
| 730 | return [new_repo] |
| 731 | if query.startswith("pushed:"): |
| 732 | return [trending_repo] |
| 733 | raise AssertionError(f"Unexpected query: {query}") |
| 734 | |
| 735 | def has_readme(self, full_name: str) -> bool: |
| 736 | return True |
| 737 | |
| 738 | args = Namespace( |
| 739 | since="2026-05-11", |
| 740 | as_of="2026-05-18", |
| 741 | max_results=10, |
| 742 | output=str(output_path), |
| 743 | topic=None, |
| 744 | config=None, |
| 745 | ) |
| 746 | |
| 747 | with ( |
| 748 | mock.patch.object(crawl, "parse_args", return_value=args), |
| 749 | mock.patch.dict("os.environ", {"GITHUB_TOKEN": "token"}, clear=False), |
| 750 | mock.patch.object(crawl, "GitHubClient", FakeClient), |
| 751 | mock.patch.object( |
| 752 | crawl, "load_previous_star_snapshot", return_value={"octo/momentum-watch": 145} |
| 753 | ), |
| 754 | mock.patch.object(crawl, "utc_now", return_value=FIXED_RUN_TIME), |
| 755 | mock.patch.object(crawl, "snapshots_dir", return_value=snapshot_dir), |
| 756 | ): |
| 757 | exit_code = crawl.main() |
| 758 | |
| 759 | self.assertEqual(exit_code, 0) |
| 760 | payload = json.loads(output_path.read_text(encoding="utf-8")) |
| 761 | crawl.validate_payload(payload) |
| 762 | self.assertEqual(payload["week"], "2026-W21") |
| 763 | self.assertEqual(payload["trending_repos"][0]["stars_gained"], 35) |
| 764 | self.assertTrue((snapshot_dir / "2026-W21-stars.json").exists()) |
| 765 | |
| 766 | def test_generate_content_produces_valid_hugo_content(self) -> None: |
| 767 | tests_root = Path(__file__).resolve().parent |
| 768 | with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir: |
| 769 | base = Path(tmpdir) |
| 770 | summary_path = base / "data" / "analyzed" / "2026-W21-summary.md" |
| 771 | summary_path.parent.mkdir(parents=True) |
| 772 | summary_path.write_text(make_analysis_markdown(), encoding="utf-8") |
| 773 | |
| 774 | previous_cwd = Path.cwd() |
| 775 | try: |
| 776 | import os |
| 777 | |
| 778 | os.chdir(base) |
| 779 | output_path = generate_content.generate_content(summary_path) |
| 780 | finally: |
| 781 | os.chdir(previous_cwd) |
| 782 | |
| 783 | self.assertEqual(output_path, base / "content" / "weekly" / "2026" / "W21.md") |
| 784 | rendered = output_path.read_text(encoding="utf-8") |
| 785 | self.assertIn('title: "Reliable Automation Gains Ground"', rendered) |
| 786 | self.assertIn('week: "2026-W21"', rendered) |
| 787 | self.assertIn("draft: false", rendered) |
| 788 | self.assertNotIn("quality_score", rendered) |
| 789 | self.assertIn("## This Week's Trends", rendered) |
| 790 | |
| 791 | def test_analyze_fallback_no_ai_can_process_raw_data(self) -> None: |
| 792 | tests_root = Path(__file__).resolve().parent |
| 793 | with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir: |
| 794 | base = Path(tmpdir) |
| 795 | raw_path = base / "data" / "raw" / "2026-W21.json" |
| 796 | output_path = base / "data" / "analyzed" / "2026-W21-summary.md" |
| 797 | raw_path.parent.mkdir(parents=True) |
| 798 | output_path.parent.mkdir(parents=True) |
| 799 | raw_path.write_text(json.dumps(make_raw_payload()), encoding="utf-8") |
| 800 | |
| 801 | with mock.patch.object(analyze_fallback.request, "urlopen") as urlopen_mock: |
| 802 | exit_code = analyze_fallback.main( |
| 803 | [ |
| 804 | "--raw-json", |
| 805 | str(raw_path), |
| 806 | "--output", |
| 807 | str(output_path), |
| 808 | "--current-datetime", |
| 809 | FIXED_RUN_DATETIME, |
| 810 | "--analyzed-dir", |
| 811 | str(output_path.parent), |
| 812 | "--no-ai", |
| 813 | ] |
| 814 | ) |
| 815 | |
| 816 | self.assertEqual(exit_code, 0) |
| 817 | written = output_path.read_text(encoding="utf-8") |
| 818 | self.assertIn("Automation, Observability, and This Week's Repo Signals", written) |
| 819 | self.assertIn("## Signal & Noise", written) |
| 820 | urlopen_mock.assert_not_called() |
| 821 | |
| 822 | def test_analysis_gate_validates_analysis_output_correctly(self) -> None: |
| 823 | tests_root = Path(__file__).resolve().parent |
| 824 | with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir: |
| 825 | base = Path(tmpdir) |
| 826 | raw_path = base / "data" / "raw" / "2026-W21.json" |
| 827 | raw_path.parent.mkdir(parents=True) |
| 828 | raw_path.write_text(json.dumps(make_raw_payload()), encoding="utf-8") |
| 829 | |
| 830 | valid_path = base / "data" / "analyzed" / "2026-W21-summary.md" |
| 831 | valid_path.parent.mkdir(parents=True) |
| 832 | valid_path.write_text(make_analysis_markdown(), encoding="utf-8") |
| 833 | |
| 834 | self.assertEqual( |
| 835 | analysis_gate.main( |
| 836 | [ |
| 837 | "--analysis-file", |
| 838 | str(valid_path), |
| 839 | "--raw-json", |
| 840 | str(raw_path), |
| 841 | "--current-datetime", |
| 842 | FIXED_RUN_DATETIME, |
| 843 | "--source", |
| 844 | "copilot-cli", |
| 845 | ] |
| 846 | ), |
| 847 | 0, |
| 848 | ) |
| 849 | |
| 850 | # quality_score is now pipeline-owned: a hand-set value is overwritten by the |
| 851 | # deterministic objective score, so an otherwise-valid summary still passes even |
| 852 | # when the authored score is low (jmservera/SquadScope#583). |
| 853 | overwritten_path = base / "data" / "analyzed" / "overwritten-summary.md" |
| 854 | overwritten_path.write_text( |
| 855 | make_analysis_markdown().replace("quality_score: 86", "quality_score: 40"), |
| 856 | encoding="utf-8", |
| 857 | ) |
| 858 | self.assertEqual( |
| 859 | analysis_gate.main( |
| 860 | [ |
| 861 | "--analysis-file", |
| 862 | str(overwritten_path), |
| 863 | "--raw-json", |
| 864 | str(raw_path), |
| 865 | "--current-datetime", |
| 866 | FIXED_RUN_DATETIME, |
| 867 | "--source", |
| 868 | "copilot-cli", |
| 869 | ] |
| 870 | ), |
| 871 | 0, |
| 872 | ) |
| 873 | rewritten = overwritten_path.read_text(encoding="utf-8") |
| 874 | self.assertNotIn("quality_score: 40", rewritten) |
| 875 | self.assertRegex(rewritten, r"(?m)^quality_score: (?:6[0-9]|[7-9][0-9]|100)$") |
| 876 | |
| 877 | # A genuine gate violation (missing required section heading) must still be rejected. |
| 878 | invalid_path = base / "data" / "analyzed" / "invalid-summary.md" |
| 879 | invalid_path.write_text( |
| 880 | make_analysis_markdown().replace("## The Week Ahead", "## Looking Forward"), |
| 881 | encoding="utf-8", |
| 882 | ) |
| 883 | |
| 884 | with self.assertRaises(SystemExit) as exc: |
| 885 | analysis_gate.main( |
| 886 | [ |
| 887 | "--analysis-file", |
| 888 | str(invalid_path), |
| 889 | "--raw-json", |
| 890 | str(raw_path), |
| 891 | "--current-datetime", |
| 892 | FIXED_RUN_DATETIME, |
| 893 | "--source", |
| 894 | "copilot-cli", |
| 895 | ] |
| 896 | ) |
| 897 | |
| 898 | self.assertEqual(exc.exception.code, 1) |
| 899 | |
| 900 | |
| 901 | if __name__ == "__main__": |
| 902 | unittest.main() |