| 1 | """Automated QA gates for matrix crawl and map/reduce acceptance criteria. |
| 2 | |
| 3 | Covers: |
| 4 | - Reducer correctness: deterministic fan-in, citation preservation, contradiction handling |
| 5 | - End-to-end dry-run: full pipeline on representative fixtures with gate validation |
| 6 | - Cost/token guardrails: budget accounting, bounds enforcement |
| 7 | - Failure handling: mapper/reducer failure paths and fallback behavior |
| 8 | |
| 9 | These gates run as part of the existing pytest CI path (issue #438). |
| 10 | """ |
| 11 | |
| 12 | from __future__ import annotations |
| 13 | |
| 14 | import json |
| 15 | from pathlib import Path |
| 16 | |
| 17 | import pytest |
| 18 | |
| 19 | from scripts import map_reduce_dry_run as dry_run |
| 20 | from scripts.model_pricing import MODEL_RATES, estimate_cost_usd |
| 21 | from scripts.preflight_cost_check import estimate_input_tokens |
| 22 | from scripts.preflight_cost_check import main as preflight_main |
| 23 | |
| 24 | # --------------------------------------------------------------------------- |
| 25 | # Fixtures |
| 26 | # --------------------------------------------------------------------------- |
| 27 | |
| 28 | |
| 29 | def make_repo(owner: str, name: str, stars: int, gained: int = 0, **extra) -> dict: |
| 30 | base = { |
| 31 | "name": name, |
| 32 | "owner": owner, |
| 33 | "full_name": f"{owner}/{name}", |
| 34 | "description": f"{name} delivers evidence-backed developer tooling for the open-source ecosystem.", |
| 35 | "language": "Python", |
| 36 | "stars": stars, |
| 37 | "stars_gained": gained, |
| 38 | "created_at": "2026-05-18T10:00:00Z", |
| 39 | "topics": ["ai", "developer-tools"], |
| 40 | "url": f"https://github.com/{owner}/{name}", |
| 41 | } |
| 42 | base.update(extra) |
| 43 | return base |
| 44 | |
| 45 | |
| 46 | def valid_finding(claim_id: str = "claim-1", repo: str = "org/repo", **overrides) -> dict: |
| 47 | base = { |
| 48 | "claim_id": claim_id, |
| 49 | "claim": f"{repo} is supported by repository evidence.", |
| 50 | "category": "trend", |
| 51 | "source_type": "github", |
| 52 | "evidence_refs": [{"type": "repo", "ref": repo, "url": f"https://github.com/{repo}"}], |
| 53 | "repo_full_name": repo, |
| 54 | "news_url": None, |
| 55 | "confidence": 0.78, |
| 56 | "contra_refs": [], |
| 57 | "uncertainties": [], |
| 58 | } |
| 59 | base.update(overrides) |
| 60 | return base |
| 61 | |
| 62 | |
| 63 | def valid_ledger(findings=None, shard_id="signal-type:test") -> dict: |
| 64 | if findings is None: |
| 65 | findings = [valid_finding()] |
| 66 | return { |
| 67 | "schema_version": "analysis_map_v1", |
| 68 | "run_id": "local", |
| 69 | "week": "2026-W21", |
| 70 | "shard_id": shard_id, |
| 71 | "slice": {}, |
| 72 | "coverage": { |
| 73 | "repo_ids_seen": ["org/repo"], |
| 74 | "article_urls_seen": [], |
| 75 | "repo_count_input": 1, |
| 76 | "repo_count_mapped": 1, |
| 77 | "article_count_input": 0, |
| 78 | "article_count_mapped": 0, |
| 79 | "excluded_reason_counts": {}, |
| 80 | }, |
| 81 | "findings": findings, |
| 82 | "citations": [], |
| 83 | "reference_candidates": {"notable_projects": ["org/repo"], "press_articles": []}, |
| 84 | "provenance": {}, |
| 85 | } |
| 86 | |
| 87 | |
| 88 | RAW_PAYLOAD = { |
| 89 | "week": "2026-W21", |
| 90 | "crawled_at": "2026-05-20T12:00:00Z", |
| 91 | "new_repos": [make_repo("octo", "alpha", 1200, 50), make_repo("octo", "beta", 900, 30)], |
| 92 | "trending_repos": [ |
| 93 | make_repo("tools", "gamma", 5000, 450), |
| 94 | make_repo("tools", "delta", 3000, 250), |
| 95 | ], |
| 96 | "signals": {"top_topics": ["ai", "developer-tools", "testing"]}, |
| 97 | } |
| 98 | |
| 99 | PRESS_CONTEXT = ( |
| 100 | "### Correlation Summary\n" |
| 101 | "- Industry article: https://example.com/ai-tooling links repo momentum to developer tools.\n" |
| 102 | "- Industry article: https://example.com/oss-growth shows OSS ecosystem expansion.\n" |
| 103 | ) |
| 104 | |
| 105 | |
| 106 | # =========================================================================== |
| 107 | # Section 1: Reducer correctness |
| 108 | # =========================================================================== |
| 109 | |
| 110 | |
| 111 | class TestReducerDeterministicFanIn: |
| 112 | """Validates deterministic fan-in assumptions needed by dry-run map/reduce.""" |
| 113 | |
| 114 | def test_identical_ledgers_produce_stable_output(self): |
| 115 | """Running reduce_ledgers twice with same input produces identical plan.""" |
| 116 | ledger = valid_ledger() |
| 117 | plan1, rej1, con1 = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 118 | plan2, rej2, con2 = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 119 | |
| 120 | assert plan1 == plan2 |
| 121 | assert rej1 == rej2 |
| 122 | assert con1 == con2 |
| 123 | |
| 124 | def test_ledger_ordering_is_deterministic(self): |
| 125 | """Reducer produces same output regardless of ledger insertion order.""" |
| 126 | ledger_a = valid_ledger( |
| 127 | [valid_finding("a1", "org/alpha")], |
| 128 | shard_id="signal-type:new_repos", |
| 129 | ) |
| 130 | ledger_b = valid_ledger( |
| 131 | [valid_finding("b1", "org/beta")], |
| 132 | shard_id="signal-type:trending_repos", |
| 133 | ) |
| 134 | |
| 135 | plan_ab, _, _ = dry_run.reduce_ledgers([ledger_a, ledger_b], raw_payload=RAW_PAYLOAD) |
| 136 | plan_ba, _, _ = dry_run.reduce_ledgers([ledger_b, ledger_a], raw_payload=RAW_PAYLOAD) |
| 137 | |
| 138 | # Selected claims sorted by section/confidence, so order should be stable |
| 139 | assert [c["claim_id"] for c in plan_ab["selected_claims"]] == [ |
| 140 | c["claim_id"] for c in plan_ba["selected_claims"] |
| 141 | ] |
| 142 | |
| 143 | def test_duplicate_claims_collapsed(self): |
| 144 | """Claims with the same normalized key are deduplicated by the reducer.""" |
| 145 | finding = valid_finding("dup-1", "org/repo") |
| 146 | finding_dup = valid_finding("dup-2", "org/repo") |
| 147 | ledger = valid_ledger([finding, finding_dup]) |
| 148 | |
| 149 | plan, rejected, _ = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 150 | |
| 151 | selected_ids = [c["claim_id"] for c in plan["selected_claims"]] |
| 152 | assert len(selected_ids) == 1 |
| 153 | dup_rejects = [r for r in rejected if r["reason"] == "duplicate"] |
| 154 | assert len(dup_rejects) == 1 |
| 155 | |
| 156 | |
| 157 | class TestReducerCitationPreservation: |
| 158 | """Validates citation preservation for mapper/reducer outputs.""" |
| 159 | |
| 160 | def test_selected_claims_retain_citation_bindings(self): |
| 161 | """Selected claims keep repo and article citation bindings through reduce.""" |
| 162 | finding = valid_finding("cit-1", "org/cited") |
| 163 | finding["evidence_refs"].append( |
| 164 | { |
| 165 | "type": "article", |
| 166 | "ref": "https://news.example.com/1", |
| 167 | "url": "https://news.example.com/1", |
| 168 | } |
| 169 | ) |
| 170 | ledger = valid_ledger([finding]) |
| 171 | |
| 172 | plan, _, _ = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 173 | |
| 174 | assert len(plan["selected_claims"]) == 1 |
| 175 | bindings = plan["selected_claims"][0]["citation_bindings"] |
| 176 | assert "org/cited" in bindings["repos"] |
| 177 | assert "https://news.example.com/1" in bindings["articles"] |
| 178 | |
| 179 | def test_merged_claims_combine_citations(self): |
| 180 | """When claims merge, citation bindings from both are combined.""" |
| 181 | f1 = valid_finding("merge-1", "org/repo") |
| 182 | f1["evidence_refs"] = [ |
| 183 | {"type": "repo", "ref": "org/repo", "url": "https://github.com/org/repo"} |
| 184 | ] |
| 185 | |
| 186 | # Second ledger with same normalized key but different citation |
| 187 | f2 = valid_finding("merge-2", "org/repo") |
| 188 | f2["evidence_refs"] = [ |
| 189 | {"type": "repo", "ref": "org/repo", "url": "https://github.com/org/repo"}, |
| 190 | { |
| 191 | "type": "article", |
| 192 | "ref": "https://news.example.com/x", |
| 193 | "url": "https://news.example.com/x", |
| 194 | }, |
| 195 | ] |
| 196 | ledger1 = valid_ledger([f1], shard_id="signal-type:shard1") |
| 197 | ledger2 = valid_ledger([f2], shard_id="signal-type:shard2") |
| 198 | |
| 199 | plan, _, _ = dry_run.reduce_ledgers([ledger1, ledger2], raw_payload=RAW_PAYLOAD) |
| 200 | |
| 201 | assert len(plan["selected_claims"]) == 1 |
| 202 | bindings = plan["selected_claims"][0]["citation_bindings"] |
| 203 | assert "org/repo" in bindings["repos"] |
| 204 | assert "https://news.example.com/x" in bindings["articles"] |
| 205 | |
| 206 | def test_weak_citation_claims_rejected(self): |
| 207 | """Findings without evidence refs are rejected as weak_citation.""" |
| 208 | finding = valid_finding("weak-1", "org/empty") |
| 209 | finding["evidence_refs"] = [] |
| 210 | ledger = valid_ledger([finding]) |
| 211 | |
| 212 | plan, rejected, _ = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 213 | |
| 214 | assert plan["selected_claims"] == [] |
| 215 | weak = [r for r in rejected if r["reason"] == "weak_citation"] |
| 216 | assert len(weak) == 1 |
| 217 | assert weak[0]["claim_id"] == "weak-1" |
| 218 | |
| 219 | |
| 220 | class TestReducerContradictionHandling: |
| 221 | """Validates contradiction detection and rejection behavior.""" |
| 222 | |
| 223 | def test_mutual_contradiction_rejects_both(self): |
| 224 | """Claims that reference each other's contra_refs are both rejected.""" |
| 225 | f1 = valid_finding("contra-1", "org/a") |
| 226 | f1["contra_refs"] = ["contra-2"] |
| 227 | f2 = valid_finding("contra-2", "org/b") |
| 228 | f2["contra_refs"] = ["contra-1"] |
| 229 | ledger = valid_ledger([f1, f2]) |
| 230 | |
| 231 | plan, rejected, contradictions = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 232 | |
| 233 | assert plan["selected_claims"] == [] |
| 234 | contra_ids = {c["claim_id"] for c in contradictions} |
| 235 | assert contra_ids == {"contra-1", "contra-2"} |
| 236 | reject_ids = {r["claim_id"] for r in rejected if r["reason"] == "unresolved_contradiction"} |
| 237 | assert reject_ids == {"contra-1", "contra-2"} |
| 238 | |
| 239 | def test_one_sided_contradiction_rejects_contradicted_claim(self): |
| 240 | """A claim that contradicts another is rejected along with its target.""" |
| 241 | f1 = valid_finding("target-1", "org/target") |
| 242 | f2 = valid_finding("attacker-1", "org/attacker") |
| 243 | f2["contra_refs"] = ["target-1"] |
| 244 | ledger = valid_ledger([f1, f2]) |
| 245 | |
| 246 | plan, rejected, contradictions = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 247 | |
| 248 | # The attacker has contra_refs, so it's contradicted |
| 249 | # The target is contradicted_by the attacker |
| 250 | contra_ids = {c["claim_id"] for c in contradictions} |
| 251 | assert "attacker-1" in contra_ids |
| 252 | assert "target-1" in contra_ids |
| 253 | |
| 254 | def test_contradictions_sidecar_populated(self): |
| 255 | """Contradictions list is preserved in the plan for audit.""" |
| 256 | f1 = valid_finding("sc-1", "org/x") |
| 257 | f1["contra_refs"] = ["sc-2"] |
| 258 | f2 = valid_finding("sc-2", "org/y") |
| 259 | ledger = valid_ledger([f1, f2]) |
| 260 | |
| 261 | plan, _, contradictions = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 262 | |
| 263 | assert plan["contradictions"] == contradictions |
| 264 | assert len(contradictions) > 0 |
| 265 | for c in contradictions: |
| 266 | assert "claim_id" in c |
| 267 | assert "resolution" in c |
| 268 | assert c["resolution"] == "rejected_unresolved" |
| 269 | |
| 270 | |
| 271 | # =========================================================================== |
| 272 | # Section 2: End-to-end dry-run tests |
| 273 | # =========================================================================== |
| 274 | |
| 275 | |
| 276 | class TestEndToEndDryRun: |
| 277 | """Exercises full map/reduce scaffolding on fixtures and validates gate outcomes.""" |
| 278 | |
| 279 | @pytest.fixture |
| 280 | def workspace(self, tmp_path): |
| 281 | """Create representative fixture workspace.""" |
| 282 | raw_path = tmp_path / "data" / "raw" / "2026-W21.json" |
| 283 | press_path = tmp_path / "data" / "analyzed" / "2026-W21-press-context.md" |
| 284 | output_dir = tmp_path / "data" / "candidates" / "2026-W21" / "local" / "map-reduce" |
| 285 | raw_path.parent.mkdir(parents=True) |
| 286 | press_path.parent.mkdir(parents=True) |
| 287 | raw_path.write_text(json.dumps(RAW_PAYLOAD), encoding="utf-8") |
| 288 | press_path.write_text(PRESS_CONTEXT, encoding="utf-8") |
| 289 | return raw_path, press_path, output_dir |
| 290 | |
| 291 | def test_full_pipeline_produces_passing_qa(self, workspace): |
| 292 | """End-to-end run produces qa-comparison-report with status=passed.""" |
| 293 | raw_path, press_path, output_dir = workspace |
| 294 | |
| 295 | rc = dry_run.main( |
| 296 | [ |
| 297 | "--raw-json", |
| 298 | raw_path.as_posix(), |
| 299 | "--press-context", |
| 300 | press_path.as_posix(), |
| 301 | "--output-dir", |
| 302 | output_dir.as_posix(), |
| 303 | "--current-datetime", |
| 304 | "2026-05-20T12:00:00Z", |
| 305 | "--run-id", |
| 306 | "qa-gate-test", |
| 307 | ] |
| 308 | ) |
| 309 | |
| 310 | assert rc == 0 |
| 311 | qa = json.loads((output_dir / "qa-comparison-report.json").read_text(encoding="utf-8")) |
| 312 | assert qa["status"] == "passed" |
| 313 | assert qa["publish_eligible"] is False |
| 314 | |
| 315 | def test_all_mapper_contracts_valid(self, workspace): |
| 316 | """Each mapper ledger passes validate_map with zero errors.""" |
| 317 | raw_path, press_path, output_dir = workspace |
| 318 | dry_run.main( |
| 319 | [ |
| 320 | "--raw-json", |
| 321 | raw_path.as_posix(), |
| 322 | "--press-context", |
| 323 | press_path.as_posix(), |
| 324 | "--output-dir", |
| 325 | output_dir.as_posix(), |
| 326 | "--current-datetime", |
| 327 | "2026-05-20T12:00:00Z", |
| 328 | "--run-id", |
| 329 | "qa-gate-test", |
| 330 | ] |
| 331 | ) |
| 332 | |
| 333 | for mapper in dry_run.MAPPER_IDS: |
| 334 | ledger = json.loads( |
| 335 | (output_dir / "maps" / f"{mapper}.json").read_text(encoding="utf-8") |
| 336 | ) |
| 337 | errors = dry_run.validate_map(ledger) |
| 338 | assert errors == [], f"Mapper {mapper} contract errors: {errors}" |
| 339 | |
| 340 | def test_sidecars_always_present(self, workspace): |
| 341 | """rejected-claims.json and contradictions.json are always emitted.""" |
| 342 | raw_path, press_path, output_dir = workspace |
| 343 | dry_run.main( |
| 344 | [ |
| 345 | "--raw-json", |
| 346 | raw_path.as_posix(), |
| 347 | "--press-context", |
| 348 | press_path.as_posix(), |
| 349 | "--output-dir", |
| 350 | output_dir.as_posix(), |
| 351 | "--current-datetime", |
| 352 | "2026-05-20T12:00:00Z", |
| 353 | "--run-id", |
| 354 | "qa-gate-test", |
| 355 | ] |
| 356 | ) |
| 357 | |
| 358 | rejected = json.loads( |
| 359 | (output_dir / "sidecars" / "rejected-claims.json").read_text(encoding="utf-8") |
| 360 | ) |
| 361 | contras = json.loads( |
| 362 | (output_dir / "sidecars" / "contradictions.json").read_text(encoding="utf-8") |
| 363 | ) |
| 364 | assert rejected["schema_version"] == "analysis_rejected_claims_v1" |
| 365 | assert contras["schema_version"] == "analysis_contradictions_v1" |
| 366 | assert isinstance(rejected["rejected_claims"], list) |
| 367 | assert isinstance(contras["contradictions"], list) |
| 368 | |
| 369 | def test_manifest_is_never_publish_eligible(self, workspace): |
| 370 | """Dry-run manifest always marks candidate_only=True, publish_eligible=False.""" |
| 371 | raw_path, press_path, output_dir = workspace |
| 372 | dry_run.main( |
| 373 | [ |
| 374 | "--raw-json", |
| 375 | raw_path.as_posix(), |
| 376 | "--press-context", |
| 377 | press_path.as_posix(), |
| 378 | "--output-dir", |
| 379 | output_dir.as_posix(), |
| 380 | "--current-datetime", |
| 381 | "2026-05-20T12:00:00Z", |
| 382 | "--run-id", |
| 383 | "qa-gate-test", |
| 384 | ] |
| 385 | ) |
| 386 | |
| 387 | manifest = json.loads((output_dir / "manifest.json").read_text(encoding="utf-8")) |
| 388 | assert manifest["publish_eligible"] is False |
| 389 | assert manifest["candidate_only"] is True |
| 390 | |
| 391 | def test_qa_report_documents_expected_provenance_failure(self, workspace): |
| 392 | """The provenance gate fails as expected (dry-run is not publishable AI).""" |
| 393 | raw_path, press_path, output_dir = workspace |
| 394 | dry_run.main( |
| 395 | [ |
| 396 | "--raw-json", |
| 397 | raw_path.as_posix(), |
| 398 | "--press-context", |
| 399 | press_path.as_posix(), |
| 400 | "--output-dir", |
| 401 | output_dir.as_posix(), |
| 402 | "--current-datetime", |
| 403 | "2026-05-20T12:00:00Z", |
| 404 | "--run-id", |
| 405 | "qa-gate-test", |
| 406 | ] |
| 407 | ) |
| 408 | |
| 409 | qa = json.loads((output_dir / "qa-comparison-report.json").read_text(encoding="utf-8")) |
| 410 | provenance = qa["checks"]["publish_provenance_gate"] |
| 411 | assert provenance["passed"] is False |
| 412 | assert provenance["expected_failure"] is True |
| 413 | |
| 414 | def test_candidate_markdown_contains_repo_links(self, workspace): |
| 415 | """Candidate markdown includes hyperlinks to featured repositories.""" |
| 416 | raw_path, press_path, output_dir = workspace |
| 417 | dry_run.main( |
| 418 | [ |
| 419 | "--raw-json", |
| 420 | raw_path.as_posix(), |
| 421 | "--press-context", |
| 422 | press_path.as_posix(), |
| 423 | "--output-dir", |
| 424 | output_dir.as_posix(), |
| 425 | "--current-datetime", |
| 426 | "2026-05-20T12:00:00Z", |
| 427 | "--run-id", |
| 428 | "qa-gate-test", |
| 429 | ] |
| 430 | ) |
| 431 | |
| 432 | candidate = (output_dir / "2026-W21-map-reduce-candidate.md").read_text(encoding="utf-8") |
| 433 | assert "[tools/gamma](https://github.com/tools/gamma)" in candidate |
| 434 | assert "Map/reduce dry-run candidate only" in candidate |
| 435 | |
| 436 | |
| 437 | # =========================================================================== |
| 438 | # Section 3: Cost/token guardrails |
| 439 | # =========================================================================== |
| 440 | |
| 441 | |
| 442 | class TestCostTokenGuardrails: |
| 443 | """Enforces PRD guardrails for token usage and budget accounting.""" |
| 444 | |
| 445 | def test_preflight_rejects_over_budget(self, tmp_path): |
| 446 | """Cost check fails when token budget exceeds hard cap.""" |
| 447 | big_file = tmp_path / "big.json" |
| 448 | big_file.write_text("x" * 2_000_000, encoding="utf-8") |
| 449 | |
| 450 | rc = preflight_main(["--context-files", str(big_file)]) |
| 451 | assert rc == 1 |
| 452 | |
| 453 | def test_preflight_passes_under_budget(self, tmp_path): |
| 454 | """Cost check passes when under hard cap.""" |
| 455 | small_file = tmp_path / "small.json" |
| 456 | small_file.write_text("x" * 400, encoding="utf-8") |
| 457 | |
| 458 | rc = preflight_main(["--context-files", str(small_file)]) |
| 459 | assert rc == 0 |
| 460 | |
| 461 | def test_preflight_custom_cap_enforcement(self, tmp_path): |
| 462 | """Custom hard cap correctly triggers failure.""" |
| 463 | medium_file = tmp_path / "medium.json" |
| 464 | medium_file.write_text("x" * 40_000, encoding="utf-8") |
| 465 | |
| 466 | rc = preflight_main(["--context-files", str(medium_file), "--hard-cap", "0.001"]) |
| 467 | assert rc == 1 |
| 468 | |
| 469 | def test_unknown_model_always_fails(self, tmp_path): |
| 470 | """Unknown model name causes preflight to fail rather than pass silently.""" |
| 471 | f = tmp_path / "a.json" |
| 472 | f.write_text("content", encoding="utf-8") |
| 473 | |
| 474 | rc = preflight_main(["--context-files", str(f), "--model", "no-such-model-xyz"]) |
| 475 | assert rc == 1 |
| 476 | |
| 477 | def test_token_estimate_deterministic(self, tmp_path): |
| 478 | """Token estimation is deterministic for the same input.""" |
| 479 | f = tmp_path / "stable.json" |
| 480 | f.write_text("hello world " * 100, encoding="utf-8") |
| 481 | |
| 482 | t1 = estimate_input_tokens([f]) |
| 483 | t2 = estimate_input_tokens([f]) |
| 484 | assert t1 == t2 |
| 485 | assert t1 > 0 |
| 486 | |
| 487 | def test_missing_file_yields_zero_tokens(self): |
| 488 | """Missing files contribute zero tokens, don't crash.""" |
| 489 | tokens = estimate_input_tokens([Path("/nonexistent/file.json")]) |
| 490 | assert tokens == 0 |
| 491 | |
| 492 | def test_manifest_token_estimate_is_positive(self, tmp_path): |
| 493 | """Dry-run manifest includes positive rendered_prompt_estimate tokens.""" |
| 494 | raw_path = tmp_path / "raw.json" |
| 495 | press_path = tmp_path / "press.md" |
| 496 | output_dir = tmp_path / "out" |
| 497 | raw_path.write_text(json.dumps(RAW_PAYLOAD), encoding="utf-8") |
| 498 | press_path.write_text(PRESS_CONTEXT, encoding="utf-8") |
| 499 | |
| 500 | dry_run.main( |
| 501 | [ |
| 502 | "--raw-json", |
| 503 | raw_path.as_posix(), |
| 504 | "--press-context", |
| 505 | press_path.as_posix(), |
| 506 | "--output-dir", |
| 507 | output_dir.as_posix(), |
| 508 | "--current-datetime", |
| 509 | "2026-05-20T12:00:00Z", |
| 510 | "--run-id", |
| 511 | "cost-test", |
| 512 | ] |
| 513 | ) |
| 514 | |
| 515 | manifest = json.loads((output_dir / "manifest.json").read_text(encoding="utf-8")) |
| 516 | est = manifest["component_estimates"]["rendered_prompt_estimate"] |
| 517 | assert est["tokens"] > 0 |
| 518 | assert est["bytes"] > 0 |
| 519 | assert len(est["checksum_sha256"]) == 64 |
| 520 | |
| 521 | def test_known_models_have_rates(self): |
| 522 | """All known models in MODEL_RATES produce non-None cost estimates.""" |
| 523 | for model_name in MODEL_RATES: |
| 524 | cost = estimate_cost_usd(model_name, 1000, 500) |
| 525 | assert cost is not None, f"Model {model_name} returned None cost" |
| 526 | assert cost >= 0 |
| 527 | |
| 528 | |
| 529 | # =========================================================================== |
| 530 | # Section 4: Failure handling and fallback behavior |
| 531 | # =========================================================================== |
| 532 | |
| 533 | |
| 534 | class TestMapperFailureHandling: |
| 535 | """Tests mapper and reducer failure paths.""" |
| 536 | |
| 537 | def test_validate_map_catches_missing_required_fields(self): |
| 538 | """validate_map returns errors for missing required mapper fields.""" |
| 539 | payload = {"schema_version": "analysis_map_v1"} |
| 540 | errors = dry_run.validate_map(payload) |
| 541 | missing_fields = { |
| 542 | "run_id", |
| 543 | "week", |
| 544 | "shard_id", |
| 545 | "slice", |
| 546 | "coverage", |
| 547 | "findings", |
| 548 | "citations", |
| 549 | "reference_candidates", |
| 550 | "provenance", |
| 551 | } |
| 552 | for field in missing_fields: |
| 553 | assert any(field in e for e in errors), f"Missing error for {field}" |
| 554 | |
| 555 | def test_validate_map_catches_wrong_schema_version(self): |
| 556 | """Wrong schema version produces a validation error.""" |
| 557 | ledger = valid_ledger() |
| 558 | ledger["schema_version"] = "wrong_version" |
| 559 | errors = dry_run.validate_map(ledger) |
| 560 | assert "mapper schema_version mismatch" in errors |
| 561 | |
| 562 | def test_validate_map_catches_failed_status(self): |
| 563 | """A ledger with status=failed is flagged.""" |
| 564 | ledger = valid_ledger() |
| 565 | ledger["status"] = "failed" |
| 566 | errors = dry_run.validate_map(ledger) |
| 567 | assert "mapper status failed" in errors |
| 568 | |
| 569 | def test_malformed_finding_in_reducer(self): |
| 570 | """Non-dict findings are rejected as malformed.""" |
| 571 | ledger = valid_ledger() |
| 572 | ledger["findings"] = ["not-a-dict", 42] |
| 573 | |
| 574 | plan, rejected, _ = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 575 | |
| 576 | malformed = [r for r in rejected if r["reason"] == "malformed_finding"] |
| 577 | assert len(malformed) == 2 |
| 578 | |
| 579 | def test_empty_ledger_list_produces_empty_plan(self): |
| 580 | """Reducing zero ledgers produces an empty selected_claims list.""" |
| 581 | plan, rejected, contradictions = dry_run.reduce_ledgers([], raw_payload=RAW_PAYLOAD) |
| 582 | assert plan["selected_claims"] == [] |
| 583 | assert rejected == [] |
| 584 | assert contradictions == [] |
| 585 | |
| 586 | def test_partial_press_mapper_handles_no_urls(self, tmp_path): |
| 587 | """Press mapper with no URLs emits partial status, not a crash.""" |
| 588 | press_path = tmp_path / "empty-press.md" |
| 589 | press_path.write_text("No links here.\n", encoding="utf-8") |
| 590 | raw_path = tmp_path / "raw.json" |
| 591 | raw_path.write_text(json.dumps(RAW_PAYLOAD), encoding="utf-8") |
| 592 | raw_ref = dry_run.file_ref(raw_path) |
| 593 | |
| 594 | result = dry_run.map_press( |
| 595 | run_id="test", |
| 596 | week="2026-W21", |
| 597 | press_path=press_path, |
| 598 | press_ref=dry_run.file_ref(press_path), |
| 599 | raw_ref=raw_ref, |
| 600 | ) |
| 601 | |
| 602 | assert result["status"] == "partial" |
| 603 | assert result["findings"] == [] |
| 604 | assert any("No press URLs" in e for e in result["errors"]) |
| 605 | |
| 606 | def test_end_to_end_without_press_context(self, tmp_path): |
| 607 | """Pipeline still runs when press context is unavailable.""" |
| 608 | raw_path = tmp_path / "raw.json" |
| 609 | output_dir = tmp_path / "out" |
| 610 | raw_path.write_text(json.dumps(RAW_PAYLOAD), encoding="utf-8") |
| 611 | |
| 612 | rc = dry_run.main( |
| 613 | [ |
| 614 | "--raw-json", |
| 615 | raw_path.as_posix(), |
| 616 | "--output-dir", |
| 617 | output_dir.as_posix(), |
| 618 | "--current-datetime", |
| 619 | "2026-05-20T12:00:00Z", |
| 620 | "--run-id", |
| 621 | "no-press", |
| 622 | ] |
| 623 | ) |
| 624 | |
| 625 | assert rc == 0 |
| 626 | qa = json.loads((output_dir / "qa-comparison-report.json").read_text(encoding="utf-8")) |
| 627 | # Should still pass structural checks even without press |
| 628 | assert qa["checks"]["structural_analysis_gate"]["passed"] is True |
| 629 | |
| 630 | |
| 631 | # =========================================================================== |
| 632 | # Section 5: Gate output clarity (CI-facing documentation) |
| 633 | # =========================================================================== |
| 634 | |
| 635 | |
| 636 | class TestGateOutputClarity: |
| 637 | """Validates that gate failures produce clear, actionable output.""" |
| 638 | |
| 639 | def test_validate_map_errors_are_descriptive(self): |
| 640 | """Each validation error string identifies what failed and why.""" |
| 641 | payload = { |
| 642 | "schema_version": "wrong", |
| 643 | "findings": "not-a-list", |
| 644 | } |
| 645 | errors = dry_run.validate_map(payload) |
| 646 | |
| 647 | # Every error should be a non-empty string |
| 648 | for error in errors: |
| 649 | assert isinstance(error, str) |
| 650 | assert len(error) > 10, f"Error too terse to be actionable: {error!r}" |
| 651 | |
| 652 | def test_qa_report_identifies_failing_gate(self, tmp_path): |
| 653 | """QA report structure makes it clear which gate failed.""" |
| 654 | raw_path = tmp_path / "raw.json" |
| 655 | press_path = tmp_path / "press.md" |
| 656 | output_dir = tmp_path / "out" |
| 657 | raw_path.write_text(json.dumps(RAW_PAYLOAD), encoding="utf-8") |
| 658 | press_path.write_text(PRESS_CONTEXT, encoding="utf-8") |
| 659 | |
| 660 | dry_run.main( |
| 661 | [ |
| 662 | "--raw-json", |
| 663 | raw_path.as_posix(), |
| 664 | "--press-context", |
| 665 | press_path.as_posix(), |
| 666 | "--output-dir", |
| 667 | output_dir.as_posix(), |
| 668 | "--current-datetime", |
| 669 | "2026-05-20T12:00:00Z", |
| 670 | "--run-id", |
| 671 | "clarity-test", |
| 672 | ] |
| 673 | ) |
| 674 | |
| 675 | qa = json.loads((output_dir / "qa-comparison-report.json").read_text(encoding="utf-8")) |
| 676 | # Each check section has "passed" bool and either "errors" or "expected_failure" |
| 677 | for gate_name, gate_data in qa["checks"].items(): |
| 678 | assert "passed" in gate_data or "selected" in gate_data, ( |
| 679 | f"Gate {gate_name} missing 'passed' key — CI cannot determine outcome" |
| 680 | ) |
| 681 | |
| 682 | def test_rejected_claims_include_reason(self, tmp_path): |
| 683 | """Every rejected claim has a reason field for diagnosis.""" |
| 684 | f1 = valid_finding("rej-1", "org/x") |
| 685 | f1["evidence_refs"] = [] # will be rejected as weak_citation |
| 686 | f2 = valid_finding("rej-2", "org/y") |
| 687 | f2["contra_refs"] = ["rej-1"] # will be rejected as contradiction |
| 688 | ledger = valid_ledger([f1, f2]) |
| 689 | |
| 690 | _, rejected, _ = dry_run.reduce_ledgers([ledger], raw_payload=RAW_PAYLOAD) |
| 691 | |
| 692 | for item in rejected: |
| 693 | assert "reason" in item, f"Rejected claim missing reason: {item}" |
| 694 | assert item["reason"] in { |
| 695 | "weak_citation", |
| 696 | "unresolved_contradiction", |
| 697 | "duplicate", |
| 698 | "malformed_finding", |
| 699 | }, f"Unknown rejection reason: {item['reason']}" |