| 1 | """Tests for the map/reduce comparison framework and promotion logic.""" |
| 2 | |
| 3 | from __future__ import annotations |
| 4 | |
| 5 | import json |
| 6 | from datetime import UTC, datetime |
| 7 | from types import SimpleNamespace |
| 8 | |
| 9 | import pytest |
| 10 | |
| 11 | import scripts.map_reduce_comparison as comparison |
| 12 | from scripts.map_reduce_comparison import ( |
| 13 | COMPARISON_SCHEMA, |
| 14 | PROMOTION_HARD_FLOOR_COVERAGE, |
| 15 | PROMOTION_HARD_FLOOR_QUALITY, |
| 16 | PROMOTION_MIN_COVERAGE, |
| 17 | PROMOTION_MIN_QUALITY, |
| 18 | ArtifactInfo, |
| 19 | analyze_map_reduce, |
| 20 | check_promotion_eligibility, |
| 21 | compute_evidence_coverage_from_ledgers, |
| 22 | compute_verdict, |
| 23 | generate_comparison_report, |
| 24 | main, |
| 25 | should_rollback, |
| 26 | ) |
| 27 | |
| 28 | |
| 29 | def _make_artifact_info( |
| 30 | *, |
| 31 | quality_score: int = 70, |
| 32 | gate_passed: bool = True, |
| 33 | evidence_coverage: float = 0.90, |
| 34 | citation_count: int = 12, |
| 35 | word_count: int = 1800, |
| 36 | ) -> ArtifactInfo: |
| 37 | return ArtifactInfo( |
| 38 | path="test/artifact.md", |
| 39 | sha256="abc123", |
| 40 | quality_score=quality_score, |
| 41 | gate_passed=gate_passed, |
| 42 | evidence_coverage=evidence_coverage, |
| 43 | citation_count=citation_count, |
| 44 | word_count=word_count, |
| 45 | ) |
| 46 | |
| 47 | |
| 48 | def _make_comparison_report( |
| 49 | *, |
| 50 | week: str = "2026-W24", |
| 51 | verdict: str = "pass", |
| 52 | quality_score: int = 70, |
| 53 | evidence_coverage: float = 0.90, |
| 54 | gate_regression: bool = False, |
| 55 | run_datetime: str | None = None, |
| 56 | ) -> dict: |
| 57 | if run_datetime is None: |
| 58 | run_datetime = datetime.now(UTC).isoformat() |
| 59 | return { |
| 60 | "schema_version": COMPARISON_SCHEMA, |
| 61 | "week": week, |
| 62 | "run_datetime": run_datetime, |
| 63 | "single_pass": { |
| 64 | "artifact_path": "data/analyzed/summary.md", |
| 65 | "sha256": "sp_hash", |
| 66 | "quality_score": 72, |
| 67 | "gate_passed": True, |
| 68 | "evidence_coverage": 0.92, |
| 69 | "citation_count": 14, |
| 70 | "word_count": 1850, |
| 71 | }, |
| 72 | "map_reduce": { |
| 73 | "artifact_path": "data/map-reduce-candidates/candidate.md", |
| 74 | "sha256": "mr_hash", |
| 75 | "quality_score": quality_score, |
| 76 | "gate_passed": not gate_regression, |
| 77 | "evidence_coverage": evidence_coverage, |
| 78 | "citation_count": 12, |
| 79 | "word_count": 1720, |
| 80 | "mapper_errors": {}, |
| 81 | "contradictions_resolved": 2, |
| 82 | "claims_rejected": 5, |
| 83 | }, |
| 84 | "deltas": { |
| 85 | "quality_score": quality_score - 72, |
| 86 | "evidence_coverage": evidence_coverage - 0.92, |
| 87 | "citation_count": -2, |
| 88 | "word_count": -130, |
| 89 | "gate_regression": gate_regression, |
| 90 | }, |
| 91 | "verdict": verdict, |
| 92 | "blockers": [], |
| 93 | } |
| 94 | |
| 95 | |
| 96 | class TestComputeVerdict: |
| 97 | """Tests for verdict computation logic.""" |
| 98 | |
| 99 | def test_pass_when_all_criteria_met(self): |
| 100 | sp = _make_artifact_info(quality_score=72, evidence_coverage=0.92) |
| 101 | mr = _make_artifact_info(quality_score=68, evidence_coverage=0.88) |
| 102 | deltas = {"gate_regression": False} |
| 103 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 104 | assert verdict == "pass" |
| 105 | assert blockers == [] |
| 106 | |
| 107 | def test_fail_on_gate_regression(self): |
| 108 | sp = _make_artifact_info(gate_passed=True) |
| 109 | mr = _make_artifact_info(gate_passed=False) |
| 110 | deltas = {"gate_regression": True} |
| 111 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 112 | assert verdict == "fail" |
| 113 | assert any("Gate regression" in b for b in blockers) |
| 114 | |
| 115 | def test_fail_on_low_quality_score(self): |
| 116 | sp = _make_artifact_info() |
| 117 | mr = _make_artifact_info(quality_score=55) |
| 118 | deltas = {"gate_regression": False} |
| 119 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 120 | assert verdict == "fail" |
| 121 | assert any("Quality score" in b for b in blockers) |
| 122 | |
| 123 | def test_fail_on_hard_quality_floor(self): |
| 124 | sp = _make_artifact_info() |
| 125 | mr = _make_artifact_info(quality_score=50) |
| 126 | deltas = {"gate_regression": False} |
| 127 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 128 | assert verdict == "fail" |
| 129 | assert any("hard floor" in b for b in blockers) |
| 130 | |
| 131 | def test_fail_on_low_coverage(self): |
| 132 | sp = _make_artifact_info() |
| 133 | mr = _make_artifact_info(evidence_coverage=0.80) |
| 134 | deltas = {"gate_regression": False} |
| 135 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 136 | assert verdict == "fail" |
| 137 | assert any("Evidence coverage" in b for b in blockers) |
| 138 | |
| 139 | def test_fail_on_hard_coverage_floor(self): |
| 140 | sp = _make_artifact_info() |
| 141 | mr = _make_artifact_info(evidence_coverage=0.60) |
| 142 | deltas = {"gate_regression": False} |
| 143 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 144 | assert verdict == "fail" |
| 145 | assert any("hard floor" in b for b in blockers) |
| 146 | |
| 147 | def test_fail_on_orphaned_citations(self): |
| 148 | sp = _make_artifact_info() |
| 149 | mr = _make_artifact_info() |
| 150 | deltas = {"gate_regression": False, "orphaned_citations": 1} |
| 151 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 152 | assert verdict == "fail" |
| 153 | assert any("Citation integrity failed" in b for b in blockers) |
| 154 | |
| 155 | def test_fail_on_unresolved_contradictions(self): |
| 156 | sp = _make_artifact_info() |
| 157 | mr = _make_artifact_info() |
| 158 | deltas = {"gate_regression": False, "unresolved_contradictions": 1} |
| 159 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 160 | assert verdict == "fail" |
| 161 | assert any("Contradiction handling failed" in b for b in blockers) |
| 162 | |
| 163 | def test_fail_on_invalid_rejected_claims(self): |
| 164 | sp = _make_artifact_info() |
| 165 | mr = _make_artifact_info() |
| 166 | deltas = {"gate_regression": False, "invalid_rejected_claims": 1} |
| 167 | verdict, blockers = compute_verdict(sp, mr, deltas) |
| 168 | assert verdict == "fail" |
| 169 | assert any("Claim rejection audit failed" in b for b in blockers) |
| 170 | |
| 171 | |
| 172 | class TestShouldRollback: |
| 173 | """Tests for automatic rollback trigger logic.""" |
| 174 | |
| 175 | def test_no_rollback_on_passing_report(self): |
| 176 | report = _make_comparison_report(verdict="pass") |
| 177 | rollback, reason = should_rollback(report) |
| 178 | assert rollback is False |
| 179 | assert reason == "" |
| 180 | |
| 181 | def test_rollback_on_gate_regression(self): |
| 182 | report = _make_comparison_report(gate_regression=True) |
| 183 | report["deltas"]["gate_regression"] = True |
| 184 | rollback, reason = should_rollback(report) |
| 185 | assert rollback is True |
| 186 | assert "Gate regression" in reason |
| 187 | |
| 188 | def test_rollback_on_hard_quality_floor(self): |
| 189 | report = _make_comparison_report(quality_score=50) |
| 190 | rollback, reason = should_rollback(report) |
| 191 | assert rollback is True |
| 192 | assert "hard floor" in reason |
| 193 | |
| 194 | def test_rollback_on_hard_coverage_floor(self): |
| 195 | report = _make_comparison_report(evidence_coverage=0.60) |
| 196 | rollback, reason = should_rollback(report) |
| 197 | assert rollback is True |
| 198 | assert "hard floor" in reason |
| 199 | |
| 200 | def test_rollback_on_mapper_failure(self): |
| 201 | report = _make_comparison_report() |
| 202 | report["map_reduce"]["mapper_errors"] = {"new_repos": ["schema_version mismatch"]} |
| 203 | rollback, reason = should_rollback(report) |
| 204 | assert rollback is True |
| 205 | assert "Mapper failures" in reason |
| 206 | |
| 207 | def test_no_rollback_on_empty_mapper_errors(self): |
| 208 | report = _make_comparison_report() |
| 209 | report["map_reduce"]["mapper_errors"] = {"new_repos": [], "trending_repos": []} |
| 210 | rollback, reason = should_rollback(report) |
| 211 | assert rollback is False |
| 212 | |
| 213 | |
| 214 | class TestCheckPromotionEligibility: |
| 215 | """Tests for promotion eligibility across multiple runs.""" |
| 216 | |
| 217 | def test_not_eligible_with_fewer_than_3_runs(self): |
| 218 | reports = [_make_comparison_report(week=f"2026-W{i}") for i in range(2)] |
| 219 | result = check_promotion_eligibility(reports) |
| 220 | assert result["eligible"] is False |
| 221 | assert "need ≥ 3" in result["reason"] |
| 222 | |
| 223 | def test_not_eligible_with_insufficient_passing_runs(self): |
| 224 | reports = [ |
| 225 | _make_comparison_report(week="2026-W21", verdict="pass"), |
| 226 | _make_comparison_report(week="2026-W22", verdict="fail"), |
| 227 | _make_comparison_report(week="2026-W23", verdict="fail"), |
| 228 | ] |
| 229 | result = check_promotion_eligibility(reports) |
| 230 | assert result["eligible"] is False |
| 231 | assert "Only 1/" in result["reason"] |
| 232 | |
| 233 | def test_eligible_with_3_passing_runs(self): |
| 234 | reports = [ |
| 235 | _make_comparison_report(week=f"2026-W{21 + i}", quality_score=70) for i in range(3) |
| 236 | ] |
| 237 | result = check_promotion_eligibility(reports) |
| 238 | assert result["eligible"] is True |
| 239 | assert "operator opt-in" in result["reason"] |
| 240 | |
| 241 | def test_not_eligible_with_low_average_quality(self): |
| 242 | reports = [ |
| 243 | _make_comparison_report(week=f"2026-W{21 + i}", quality_score=62) for i in range(3) |
| 244 | ] |
| 245 | result = check_promotion_eligibility(reports) |
| 246 | assert result["eligible"] is False |
| 247 | assert "Average quality" in result["reason"] |
| 248 | |
| 249 | def test_not_eligible_with_stale_runs(self): |
| 250 | old_dt = "2026-04-01T00:00:00+00:00" |
| 251 | reports = [ |
| 252 | _make_comparison_report(week=f"2026-W{13 + i}", run_datetime=old_dt, quality_score=70) |
| 253 | for i in range(3) |
| 254 | ] |
| 255 | result = check_promotion_eligibility(reports) |
| 256 | assert result["eligible"] is False |
| 257 | assert "older than 28 days" in result["reason"] |
| 258 | |
| 259 | def test_not_eligible_when_most_recent_run_failed(self): |
| 260 | reports = [ |
| 261 | _make_comparison_report(week="2026-W21", verdict="pass"), |
| 262 | _make_comparison_report(week="2026-W22", verdict="pass"), |
| 263 | _make_comparison_report(week="2026-W23", verdict="pass"), |
| 264 | _make_comparison_report(week="2026-W24", verdict="fail"), |
| 265 | ] |
| 266 | result = check_promotion_eligibility(reports) |
| 267 | assert result["eligible"] is False |
| 268 | assert "most recent runs passed" in result["reason"] |
| 269 | |
| 270 | |
| 271 | class TestGenerateComparisonReport: |
| 272 | """Tests for report generation.""" |
| 273 | |
| 274 | def test_schema_version_present(self): |
| 275 | sp = _make_artifact_info() |
| 276 | mr = _make_artifact_info() |
| 277 | report = generate_comparison_report( |
| 278 | week="2026-W24", |
| 279 | single_pass=sp, |
| 280 | map_reduce=mr, |
| 281 | map_reduce_extra={}, |
| 282 | run_datetime="2026-06-14T07:00:00Z", |
| 283 | ) |
| 284 | assert report["schema_version"] == COMPARISON_SCHEMA |
| 285 | assert report["week"] == "2026-W24" |
| 286 | |
| 287 | def test_deltas_computed_correctly(self): |
| 288 | sp = _make_artifact_info(quality_score=72, citation_count=14, word_count=1850) |
| 289 | mr = _make_artifact_info(quality_score=68, citation_count=12, word_count=1720) |
| 290 | report = generate_comparison_report( |
| 291 | week="2026-W24", |
| 292 | single_pass=sp, |
| 293 | map_reduce=mr, |
| 294 | map_reduce_extra={}, |
| 295 | run_datetime="2026-06-14T07:00:00Z", |
| 296 | ) |
| 297 | assert report["deltas"]["quality_score"] == -4 |
| 298 | assert report["deltas"]["citation_count"] == -2 |
| 299 | assert report["deltas"]["word_count"] == -130 |
| 300 | |
| 301 | def test_pass_verdict_when_criteria_met(self): |
| 302 | sp = _make_artifact_info(quality_score=72, evidence_coverage=0.92) |
| 303 | mr = _make_artifact_info(quality_score=68, evidence_coverage=0.88) |
| 304 | report = generate_comparison_report( |
| 305 | week="2026-W24", |
| 306 | single_pass=sp, |
| 307 | map_reduce=mr, |
| 308 | map_reduce_extra={}, |
| 309 | run_datetime="2026-06-14T07:00:00Z", |
| 310 | ) |
| 311 | assert report["verdict"] == "pass" |
| 312 | assert report["blockers"] == [] |
| 313 | |
| 314 | |
| 315 | class TestOperatorControls: |
| 316 | """Tests for environment variable controls.""" |
| 317 | |
| 318 | def test_default_values(self): |
| 319 | assert PROMOTION_MIN_QUALITY == 60 |
| 320 | assert PROMOTION_MIN_COVERAGE == 0.85 |
| 321 | assert PROMOTION_HARD_FLOOR_QUALITY == 55 |
| 322 | assert PROMOTION_HARD_FLOOR_COVERAGE == 0.70 |
| 323 | |
| 324 | |
| 325 | class TestArtifactLoading: |
| 326 | def test_reads_mapper_ledgers_from_maps_dir(self, tmp_path): |
| 327 | maps_dir = tmp_path / "maps" |
| 328 | maps_dir.mkdir() |
| 329 | (maps_dir / "new_repos.json").write_text( |
| 330 | json.dumps( |
| 331 | { |
| 332 | "coverage": { |
| 333 | "repo_count_input": 8, |
| 334 | "repo_count_mapped": 6, |
| 335 | "article_count_input": 2, |
| 336 | "article_count_mapped": 1, |
| 337 | } |
| 338 | } |
| 339 | ), |
| 340 | encoding="utf-8", |
| 341 | ) |
| 342 | (maps_dir / "trending_repos.json").write_text( |
| 343 | json.dumps( |
| 344 | { |
| 345 | "coverage": { |
| 346 | "repo_count_input": 2, |
| 347 | "repo_count_mapped": 2, |
| 348 | "article_count_input": 0, |
| 349 | "article_count_mapped": 0, |
| 350 | } |
| 351 | } |
| 352 | ), |
| 353 | encoding="utf-8", |
| 354 | ) |
| 355 | |
| 356 | assert compute_evidence_coverage_from_ledgers(tmp_path) == pytest.approx(0.75) |
| 357 | |
| 358 | def test_analyze_map_reduce_uses_dry_run_artifacts_and_handles_bad_qa( |
| 359 | self, |
| 360 | tmp_path, |
| 361 | monkeypatch, |
| 362 | ): |
| 363 | week = "2026-W24" |
| 364 | (tmp_path / f"{week}-map-reduce-candidate.md").write_text( |
| 365 | "---\nquality_score: 68\n---\n\n[octo/repo](https://github.com/octo/repo)\n", |
| 366 | encoding="utf-8", |
| 367 | ) |
| 368 | (tmp_path / "qa-comparison-report.json").write_text("{bad json", encoding="utf-8") |
| 369 | (tmp_path / "editorial-plan.json").write_text( |
| 370 | json.dumps( |
| 371 | { |
| 372 | "top_repo": "octo/repo", |
| 373 | "selected_claims": [ |
| 374 | { |
| 375 | "citation_bindings": { |
| 376 | "repos": ["octo/repo"], |
| 377 | "articles": [], |
| 378 | } |
| 379 | } |
| 380 | ], |
| 381 | "key_references": { |
| 382 | "notable_projects": ["octo/repo"], |
| 383 | "press_articles": [], |
| 384 | }, |
| 385 | } |
| 386 | ), |
| 387 | encoding="utf-8", |
| 388 | ) |
| 389 | sidecars_dir = tmp_path / "sidecars" |
| 390 | sidecars_dir.mkdir() |
| 391 | (sidecars_dir / "contradictions.json").write_text( |
| 392 | json.dumps({"contradictions": []}), |
| 393 | encoding="utf-8", |
| 394 | ) |
| 395 | (sidecars_dir / "rejected-claims.json").write_text( |
| 396 | json.dumps({"rejected_claims": []}), |
| 397 | encoding="utf-8", |
| 398 | ) |
| 399 | maps_dir = tmp_path / "maps" |
| 400 | maps_dir.mkdir() |
| 401 | (maps_dir / "new_repos.json").write_text( |
| 402 | json.dumps( |
| 403 | { |
| 404 | "coverage": { |
| 405 | "repo_count_input": 1, |
| 406 | "repo_count_mapped": 1, |
| 407 | "article_count_input": 0, |
| 408 | "article_count_mapped": 0, |
| 409 | } |
| 410 | } |
| 411 | ), |
| 412 | encoding="utf-8", |
| 413 | ) |
| 414 | |
| 415 | monkeypatch.setattr(comparison, "validate_analysis", lambda *_args, **_kwargs: ([], 220)) |
| 416 | monkeypatch.setattr( |
| 417 | comparison, |
| 418 | "validate_publish_quality", |
| 419 | lambda *_args, **_kwargs: ([], {}), |
| 420 | ) |
| 421 | |
| 422 | info, extra = analyze_map_reduce( |
| 423 | tmp_path, |
| 424 | {"week": week}, |
| 425 | "2026-06-14T07:00:00+00:00", |
| 426 | ) |
| 427 | |
| 428 | assert info.path.endswith(f"{week}-map-reduce-candidate.md") |
| 429 | assert info.evidence_coverage == pytest.approx(1.0) |
| 430 | assert any("QA comparison report unreadable" in err for err in extra["artifact_errors"]) |
| 431 | |
| 432 | |
| 433 | class TestMain: |
| 434 | def test_returns_nonzero_on_failed_verdict(self, monkeypatch): |
| 435 | monkeypatch.setattr(comparison, "parse_args", lambda _argv=None: SimpleNamespace()) |
| 436 | monkeypatch.setattr( |
| 437 | comparison, |
| 438 | "run", |
| 439 | lambda _args: {"week": "2026-W24", "verdict": "fail", "rollback": False}, |
| 440 | ) |
| 441 | |
| 442 | assert main([]) == 1 |