| 1 | from __future__ import annotations |
| 2 | |
| 3 | import json |
| 4 | import tempfile |
| 5 | from pathlib import Path |
| 6 | from unittest.mock import patch |
| 7 | |
| 8 | from scripts import map_reduce_dry_run as dry_run |
| 9 | from scripts.observability_metrics import validate_ledger |
| 10 | |
| 11 | |
| 12 | def make_repo(owner: str, name: str, stars: int, gained: int = 0) -> dict[str, object]: |
| 13 | return { |
| 14 | "name": name, |
| 15 | "owner": owner, |
| 16 | "full_name": f"{owner}/{name}", |
| 17 | "description": f"{name} provides evidence-backed developer infrastructure for testing map reduce analysis contracts.", |
| 18 | "language": "Python", |
| 19 | "stars": stars, |
| 20 | "stars_gained": gained, |
| 21 | "created_at": "2026-05-18T10:00:00Z", |
| 22 | "topics": ["ai", "developer-tools"], |
| 23 | "url": f"https://github.com/{owner}/{name}", |
| 24 | } |
| 25 | |
| 26 | |
| 27 | def test_dry_run_emits_valid_contract_artifacts() -> None: |
| 28 | tests_root = Path(__file__).resolve().parent |
| 29 | with tempfile.TemporaryDirectory(dir=tests_root) as tmpdir: |
| 30 | base = Path(tmpdir) |
| 31 | obs_dir = base / "observability" |
| 32 | obs_dir.mkdir(parents=True) |
| 33 | raw_path = base / "data" / "raw" / "2026-W21.json" |
| 34 | press_path = base / "data" / "analyzed" / "2026-W21-press-context.md" |
| 35 | output_dir = base / "data" / "candidates" / "2026-W21" / "local" / "map-reduce" |
| 36 | raw_path.parent.mkdir(parents=True) |
| 37 | press_path.parent.mkdir(parents=True) |
| 38 | raw_payload = { |
| 39 | "week": "2026-W21", |
| 40 | "crawled_at": "2026-05-20T12:00:00Z", |
| 41 | "new_repos": [make_repo("octo", "alpha", 1200), make_repo("octo", "beta", 900)], |
| 42 | "trending_repos": [ |
| 43 | make_repo("tools", "gamma", 5000, 450), |
| 44 | make_repo("tools", "delta", 3000, 250), |
| 45 | ], |
| 46 | "signals": {"top_topics": ["ai", "developer-tools", "testing"]}, |
| 47 | } |
| 48 | raw_path.write_text(json.dumps(raw_payload), encoding="utf-8") |
| 49 | press_path.write_text( |
| 50 | "### Correlation Summary\n- Industry article: https://example.com/ai-tooling links repo momentum to developer tools.\n", |
| 51 | encoding="utf-8", |
| 52 | ) |
| 53 | |
| 54 | with patch.object(dry_run, "DEFAULT_OBSERVABILITY_DIR", obs_dir): |
| 55 | rc = dry_run.main( |
| 56 | [ |
| 57 | "--raw-json", |
| 58 | raw_path.as_posix(), |
| 59 | "--press-context", |
| 60 | press_path.as_posix(), |
| 61 | "--output-dir", |
| 62 | output_dir.as_posix(), |
| 63 | "--current-datetime", |
| 64 | "2026-05-20T12:00:00Z", |
| 65 | "--run-id", |
| 66 | "local", |
| 67 | ] |
| 68 | ) |
| 69 | |
| 70 | assert rc == 0 |
| 71 | manifest = json.loads((output_dir / "manifest.json").read_text(encoding="utf-8")) |
| 72 | assert manifest["publish_eligible"] is False |
| 73 | assert manifest["candidate_only"] is True |
| 74 | rendered_estimate = manifest["component_estimates"]["rendered_prompt_estimate"] |
| 75 | assert set(rendered_estimate) == {"bytes", "tokens", "checksum_sha256"} |
| 76 | assert rendered_estimate["tokens"] > 0 |
| 77 | assert rendered_estimate["checksum_sha256"] |
| 78 | for mapper in dry_run.MAPPER_IDS: |
| 79 | ledger = json.loads( |
| 80 | (output_dir / "maps" / f"{mapper}.json").read_text(encoding="utf-8") |
| 81 | ) |
| 82 | assert ledger["schema_version"] == "analysis_map_v1" |
| 83 | assert ledger["coverage"]["excluded_reason_counts"] == {} |
| 84 | assert dry_run.validate_map(ledger) == [] |
| 85 | plan = json.loads((output_dir / "editorial-plan.json").read_text(encoding="utf-8")) |
| 86 | assert plan["schema_version"] == "analysis_editorial_plan_v1" |
| 87 | assert (output_dir / "sidecars" / "rejected-claims.json").exists() |
| 88 | assert (output_dir / "sidecars" / "contradictions.json").exists() |
| 89 | candidate = (output_dir / "2026-W21-map-reduce-candidate.md").read_text(encoding="utf-8") |
| 90 | assert "Map/reduce dry-run candidate only" in candidate |
| 91 | assert "[tools/gamma](https://github.com/tools/gamma)" in candidate |
| 92 | qa = json.loads((output_dir / "qa-comparison-report.json").read_text(encoding="utf-8")) |
| 93 | assert qa["status"] == "passed" |
| 94 | assert qa["publish_eligible"] is False |
| 95 | assert qa["checks"]["structural_analysis_gate"]["passed"] is True |
| 96 | assert qa["checks"]["evidence_and_editorial_gates"]["passed"] is True |
| 97 | assert qa["checks"]["publish_provenance_gate"]["expected_failure"] is True |
| 98 | observability_path = obs_dir / "2026-W21-map-reduce.json" |
| 99 | observability = json.loads(observability_path.read_text(encoding="utf-8")) |
| 100 | assert validate_ledger(observability) == [] |
| 101 | assert observability["analysis_metrics"]["reduce_stage"]["status"] == "pass" |
| 102 | assert observability["environment"]["pass_fail_counts"]["map_pass"] == 4 |
| 103 | |
| 104 | |
| 105 | def test_validate_map_rejects_citationless_findings() -> None: |
| 106 | payload = { |
| 107 | "schema_version": "analysis_map_v1", |
| 108 | "run_id": "local", |
| 109 | "week": "2026-W21", |
| 110 | "shard_id": "signal-type:new_repos", |
| 111 | "slice": {}, |
| 112 | "coverage": {"repo_ids_seen": [], "article_urls_seen": [], "excluded_reason_counts": {}}, |
| 113 | "findings": [ |
| 114 | { |
| 115 | "claim_id": "bad", |
| 116 | "claim": "unsupported", |
| 117 | "category": "trend", |
| 118 | "source_type": "github", |
| 119 | "evidence_refs": [], |
| 120 | "confidence": 0.5, |
| 121 | "contra_refs": [], |
| 122 | "uncertainties": [], |
| 123 | } |
| 124 | ], |
| 125 | "citations": [], |
| 126 | "reference_candidates": {"notable_projects": [], "press_articles": []}, |
| 127 | "provenance": {}, |
| 128 | } |
| 129 | |
| 130 | assert "finding 0 has no evidence refs" in dry_run.validate_map(payload) |
| 131 | |
| 132 | |
| 133 | def valid_ledger(findings: list[dict[str, object]] | None = None) -> dict[str, object]: |
| 134 | return { |
| 135 | "schema_version": "analysis_map_v1", |
| 136 | "run_id": "local", |
| 137 | "week": "2026-W21", |
| 138 | "shard_id": "signal-type:test", |
| 139 | "slice": {}, |
| 140 | "coverage": { |
| 141 | "repo_ids_seen": ["octo/alpha"], |
| 142 | "article_urls_seen": [], |
| 143 | "repo_count_input": 1, |
| 144 | "repo_count_mapped": 1, |
| 145 | "article_count_input": 0, |
| 146 | "article_count_mapped": 0, |
| 147 | "excluded_reason_counts": {}, |
| 148 | }, |
| 149 | "findings": findings |
| 150 | if findings is not None |
| 151 | else [ |
| 152 | { |
| 153 | "claim_id": "claim-a", |
| 154 | "claim": "octo/alpha is supported by direct repository evidence.", |
| 155 | "category": "trend", |
| 156 | "source_type": "github", |
| 157 | "evidence_refs": [ |
| 158 | {"type": "repo", "ref": "octo/alpha", "url": "https://github.com/octo/alpha"} |
| 159 | ], |
| 160 | "repo_full_name": "octo/alpha", |
| 161 | "news_url": None, |
| 162 | "confidence": 0.8, |
| 163 | "contra_refs": [], |
| 164 | "uncertainties": [], |
| 165 | } |
| 166 | ], |
| 167 | "citations": [], |
| 168 | "reference_candidates": {"notable_projects": ["octo/alpha"], "press_articles": []}, |
| 169 | "provenance": {}, |
| 170 | } |
| 171 | |
| 172 | |
| 173 | def test_validate_map_rejects_malformed_ledger() -> None: |
| 174 | payload = valid_ledger() |
| 175 | payload.pop("coverage") |
| 176 | payload["findings"] = "not-a-list" |
| 177 | |
| 178 | errors = dry_run.validate_map(payload) |
| 179 | |
| 180 | assert "mapper missing coverage" in errors |
| 181 | assert "findings must be a list" in errors |
| 182 | assert "coverage must be an object" in errors |
| 183 | |
| 184 | |
| 185 | def test_validate_map_rejects_failed_or_low_coverage() -> None: |
| 186 | payload = valid_ledger() |
| 187 | payload["coverage"] = { |
| 188 | "repo_ids_seen": ["octo/alpha"], |
| 189 | "article_urls_seen": [], |
| 190 | "repo_count_input": 3, |
| 191 | "repo_count_mapped": 1, |
| 192 | "article_count_input": 0, |
| 193 | "article_count_mapped": 1, |
| 194 | "excluded_reason_counts": {}, |
| 195 | } |
| 196 | payload["status"] = "failed" |
| 197 | |
| 198 | errors = dry_run.validate_map(payload) |
| 199 | |
| 200 | assert "coverage repo_count_mapped below repo_count_input without excluded reasons" in errors |
| 201 | assert "coverage article_count_mapped exceeds article_count_input" in errors |
| 202 | assert "mapper status failed" in errors |
| 203 | |
| 204 | |
| 205 | def test_collect_gate_failure_reasons_skips_expected_failures() -> None: |
| 206 | reasons = dry_run.collect_gate_failure_reasons( |
| 207 | { |
| 208 | "checks": { |
| 209 | "mapper_contracts": { |
| 210 | "passed": False, |
| 211 | "errors_by_mapper": {"new_repos": ["missing evidence refs"]}, |
| 212 | }, |
| 213 | "structural_analysis_gate": { |
| 214 | "passed": False, |
| 215 | "errors": ["candidate below minimum word count"], |
| 216 | }, |
| 217 | "publish_provenance_gate": { |
| 218 | "passed": False, |
| 219 | "expected_failure": True, |
| 220 | "errors": ["AI provenance metadata missing"], |
| 221 | }, |
| 222 | "sidecars_present": {"passed": False}, |
| 223 | }, |
| 224 | "regressions": ["baseline summary not found"], |
| 225 | } |
| 226 | ) |
| 227 | |
| 228 | assert "new_repos: missing evidence refs" in reasons |
| 229 | assert "candidate below minimum word count" in reasons |
| 230 | assert "baseline summary not found" in reasons |
| 231 | assert "sidecars_present failed" in reasons |
| 232 | assert "AI provenance metadata missing" not in reasons |
| 233 | |
| 234 | |
| 235 | def test_reduce_rejects_and_preserves_contradictory_claims() -> None: |
| 236 | supported = valid_ledger()["findings"][0] |
| 237 | contradictory = { |
| 238 | **supported, |
| 239 | "claim_id": "claim-b", |
| 240 | "claim": "octo/alpha evidence is contradicted by another retained source.", |
| 241 | "contra_refs": ["claim-a"], |
| 242 | "confidence": 0.9, |
| 243 | } |
| 244 | ledger = valid_ledger([supported, contradictory]) |
| 245 | |
| 246 | plan, rejected, contradictions = dry_run.reduce_ledgers( |
| 247 | [ledger], |
| 248 | raw_payload={ |
| 249 | "week": "2026-W21", |
| 250 | "new_repos": [make_repo("octo", "alpha", 1200)], |
| 251 | "trending_repos": [], |
| 252 | }, |
| 253 | ) |
| 254 | |
| 255 | assert plan["selected_claims"] == [] |
| 256 | assert [item["claim_id"] for item in contradictions] == ["claim-a", "claim-b"] |
| 257 | assert { |
| 258 | item["claim_id"] for item in rejected if item["reason"] == "unresolved_contradiction" |
| 259 | } == {"claim-a", "claim-b"} |
| 260 | assert plan["contradictions"] == contradictions |