Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
mjs 73 lines 2.71 KB
Raw
1 #!/usr/bin/env node
2 // Trace-parity comparator: node harness/parity.mjs <baseline-run-dir> <candidate-run-dir>
3 // Per entry green in BOTH runs: the sets of distinct (method, path-template)
4 // pairs must be equal, ignoring pairs allowlisted in parity-variance.json.
5 // Prints a JSON report; exit 0 = all parities hold, 1 = mismatches.
6
7 import { readFileSync, readdirSync, existsSync } from 'node:fs';
8 import { join, resolve, dirname } from 'node:path';
9 import { fileURLToPath } from 'node:url';
10
11 const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
12 const [baseDir, candDir] = process.argv.slice(2);
13 if (!baseDir || !candDir) {
14 console.error('usage: parity.mjs <baseline-run-dir> <candidate-run-dir>');
15 process.exit(2);
16 }
17
18 const variance = JSON.parse(readFileSync(join(ROOT, 'parity-variance.json'), 'utf8'));
19 const ignored = new Set((variance.entries ?? []).map((e) => e.pair));
20
21 const loadRun = (dir) => {
22 const results = readFileSync(join(dir, 'results.jsonl'), 'utf8')
23 .trim()
24 .split('\n')
25 .filter(Boolean)
26 .map((l) => JSON.parse(l));
27 const byId = new Map();
28 for (const r of results) byId.set(r.id, r);
29 return byId;
30 };
31
32 const tracePairs = (dir, id) => {
33 const file = join(dir, 'traces', `${id.replace(/[/]/g, '__')}.jsonl`);
34 const pairs = new Set();
35 if (!existsSync(file)) return pairs;
36 for (const line of readFileSync(file, 'utf8').trim().split('\n').filter(Boolean)) {
37 const rec = JSON.parse(line);
38 if (rec.m && rec.p) pairs.add(`${rec.m} ${rec.p}`);
39 }
40 return pairs;
41 };
42
43 const baseline = loadRun(baseDir);
44 const candidate = loadRun(candDir);
45
46 const report = [];
47 const ids = new Set([...baseline.keys(), ...candidate.keys()]);
48 for (const id of [...ids].sort()) {
49 const b = baseline.get(id);
50 const c = candidate.get(id);
51 if (!b || !c) {
52 report.push({
53 id,
54 parity: false,
55 reason: `missing ${b ? 'candidate' : 'baseline'} result`,
56 });
57 continue;
58 }
59 if (b.status !== 'green' || c.status !== 'green') {
60 report.push({ id, parity: false, reason: `status baseline=${b.status} candidate=${c.status}` });
61 continue;
62 }
63 const bp = tracePairs(baseDir, id);
64 const cp = tracePairs(candDir, id);
65 const onlyBaseline = [...bp].filter((p) => !cp.has(p) && !ignored.has(p));
66 const onlyCandidate = [...cp].filter((p) => !bp.has(p) && !ignored.has(p));
67 const match = onlyBaseline.length === 0 && onlyCandidate.length === 0;
68 report.push({ id, parity: match, ...(match ? {} : { onlyBaseline, onlyCandidate }) });
69 }
70
71 const green = report.filter((r) => r.parity).length;
72 console.log(JSON.stringify({ compared: report.length, parityGreen: green, ignoredPairs: ignored.size, entries: report }, null, 2));
73 process.exit(report.length > 0 && report.every((r) => r.parity) ? 0 : 1);