Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 339 lines 12.8 KB
Raw
1 /**
2 * Batches of live runs: what each run recorded, how runs are reused, and the
3 * summaries `eval.ts` prints.
4 *
5 * A batch is a directory under `results/`: `batch.json` says what was
6 * measured, `runs.jsonl` has one line per run, `logs/` the full transcripts.
7 */
8
9 import { createHash } from 'node:crypto';
10 import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
11 import { join, relative } from 'node:path';
12 import { z } from 'zod';
13 import { passAtK, passHatK, wilson } from './metrics';
14
15 export const ARMS = ['skill', 'no-skill'] as const;
16 export type Arm = (typeof ARMS)[number];
17
18 export const AGENTS = ['claude-code', 'loop'] as const;
19 export type AgentName = (typeof AGENTS)[number];
20
21 const CheckRecord = z.object({ name: z.string(), ok: z.boolean(), seen: z.string().optional() });
22
23 export const RunRecord = z.object({
24 journey: z.string(),
25 split: z.enum(['train', 'holdout']),
26 arm: z.enum(ARMS),
27 attempt: z.number(),
28 fingerprint: z.string(),
29 /**
30 * `passed` and `failed` are graded. `infra` means the run says nothing about
31 * the agent (the harness or the provider failed) and is left out of every rate.
32 */
33 outcome: z.enum(['passed', 'failed', 'infra']),
34 checks: z.array(CheckRecord),
35 /** Detours taken; never a failure, see `Journey.detours`. */
36 detours: z.array(CheckRecord),
37 stop: z.string(),
38 modelTurns: z.number(),
39 commands: z.array(z.object({ command: z.string(), exitCode: z.number() })),
40 costUsd: z.number(),
41 durationMs: z.number(),
42 error: z.string().optional(),
43 /** Set when the run was copied from an earlier batch with the same fingerprint. */
44 reusedFrom: z.string().optional(),
45 });
46 export type RunRecord = z.infer<typeof RunRecord>;
47
48 export const BatchInfo = z.object({
49 id: z.string(),
50 startedAt: z.string(),
51 agent: z.enum(AGENTS),
52 agentVersion: z.string(),
53 model: z.string(),
54 runs: z.number(),
55 arms: z.array(z.enum(ARMS)),
56 journeys: z.array(z.string()),
57 revision: z.string(),
58 /** Uncommitted changes under `ts/packages/cli` or the harness. */
59 dirty: z.boolean(),
60 skillSha256: z.string(),
61 harnessSha256: z.string(),
62 /** sha256 of the CLI binary and its companion modules, as the journey container has them. */
63 cliSha256: z.string(),
64 maxCostUsd: z.number(),
65 });
66 export type BatchInfo = z.infer<typeof BatchInfo>;
67
68 /** sha256 of the given files and of every file under the given directories, by relative path. */
69 export const treeHash = (root: string, paths: ReadonlyArray<string>): string => {
70 const hash = createHash('sha256');
71 const walk = (path: string) => {
72 if (statSync(path).isDirectory()) {
73 for (const name of readdirSync(path).sort()) {
74 if (name !== 'node_modules') walk(join(path, name));
75 }
76 return;
77 }
78 hash
79 .update(`${relative(root, path)}\0`)
80 .update(readFileSync(path))
81 .update('\0');
82 };
83 for (const path of paths) walk(join(root, path));
84 return hash.digest('hex');
85 };
86
87 /**
88 * What makes two runs interchangeable: the same journey, under the same
89 * graders, with the same CLI, agent, model, and (for the skill arm) skill.
90 */
91 export const fingerprint = (parts: {
92 journey: string;
93 arm: Arm;
94 agent: AgentName;
95 agentVersion: string;
96 model: string;
97 skillSha256: string;
98 harnessSha256: string;
99 cliSha256: string;
100 }): string =>
101 createHash('sha256')
102 .update(
103 JSON.stringify({
104 ...parts,
105 skillSha256: parts.arm === 'skill' ? parts.skillSha256 : 'none',
106 })
107 )
108 .digest('hex')
109 .slice(0, 16);
110
111 export const readRuns = (batchDirectory: string): RunRecord[] => {
112 const path = join(batchDirectory, 'runs.jsonl');
113 if (!existsSync(path)) return [];
114 return readFileSync(path, 'utf8')
115 .split('\n')
116 .filter(line => line.trim() !== '')
117 .map(line => RunRecord.parse(JSON.parse(line)));
118 };
119
120 export const readBatch = (batchDirectory: string): { info: BatchInfo; runs: RunRecord[] } => ({
121 info: BatchInfo.parse(JSON.parse(readFileSync(join(batchDirectory, 'batch.json'), 'utf8'))),
122 runs: readRuns(batchDirectory),
123 });
124
125 /** Graded runs from earlier batches, by fingerprint, oldest first. */
126 export const previousRuns = (resultsDirectory: string): Map<string, RunRecord[]> => {
127 const byFingerprint = new Map<string, RunRecord[]>();
128 if (!existsSync(resultsDirectory)) return byFingerprint;
129 for (const name of readdirSync(resultsDirectory).sort()) {
130 const directory = join(resultsDirectory, name);
131 if (!statSync(directory).isDirectory()) continue;
132 for (const run of readRuns(directory)) {
133 if (run.outcome === 'infra' || run.reusedFrom !== undefined) continue;
134 const list = byFingerprint.get(run.fingerprint) ?? [];
135 list.push({ ...run, reusedFrom: name });
136 byFingerprint.set(run.fingerprint, list);
137 }
138 }
139 return byFingerprint;
140 };
141
142 export interface CellSummary {
143 readonly journey: string;
144 readonly split: 'train' | 'holdout';
145 readonly arm: Arm;
146 /** Graded runs. */
147 readonly n: number;
148 readonly passed: number;
149 readonly infra: number;
150 readonly passAt1: number;
151 readonly interval: { low: number; high: number };
152 /** At k = the number of graded runs. */
153 readonly passAtK: number;
154 readonly passHatK: number;
155 /** Share of graded runs that took at least one detour. */
156 readonly wrongTreeRate: number;
157 readonly meanCostUsd: number;
158 readonly meanTurns: number;
159 /** How often each check failed, by name. */
160 readonly failedChecks: Readonly<Record<string, number>>;
161 }
162
163 export const summarize = (runs: ReadonlyArray<RunRecord>): CellSummary[] => {
164 const cells = new Map<string, RunRecord[]>();
165 for (const run of runs) {
166 const key = `${run.journey}\0${run.arm}`;
167 cells.set(key, [...(cells.get(key) ?? []), run]);
168 }
169 return [...cells.values()]
170 .map(cell => {
171 const [first] = cell;
172 const graded = cell.filter(run => run.outcome !== 'infra');
173 const n = graded.length;
174 const passed = graded.filter(run => run.outcome === 'passed').length;
175 const failedChecks: Record<string, number> = {};
176 for (const run of graded) {
177 for (const check of run.checks.filter(item => !item.ok)) {
178 failedChecks[check.name] = (failedChecks[check.name] ?? 0) + 1;
179 }
180 }
181 const mean = (values: number[]) =>
182 values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
183 return {
184 journey: first!.journey,
185 split: first!.split,
186 arm: first!.arm,
187 n,
188 passed,
189 infra: cell.length - n,
190 passAt1: n === 0 ? 0 : passed / n,
191 interval: wilson(passed, n),
192 passAtK: n === 0 ? 0 : passAtK(n, passed, n),
193 passHatK: n === 0 ? 0 : passHatK(n, passed, n),
194 wrongTreeRate: n === 0 ? 0 : graded.filter(run => run.detours.length > 0).length / n,
195 meanCostUsd: mean(cell.map(run => run.costUsd)),
196 meanTurns: mean(graded.map(run => run.modelTurns)),
197 failedChecks,
198 };
199 })
200 .sort((left, right) =>
201 left.split === right.split
202 ? left.journey.localeCompare(right.journey) || left.arm.localeCompare(right.arm)
203 : left.split === 'train'
204 ? -1
205 : 1
206 );
207 };
208
209 const percent = (value: number) => `${Math.round(value * 100)}%`;
210 const money = (value: number) => `$${value.toFixed(value < 1 ? 3 : 2)}`;
211
212 /** Mean over journeys, so a journey with more runs does not count for more. */
213 const splitMean = (
214 cells: ReadonlyArray<CellSummary>,
215 split: 'train' | 'holdout',
216 arm: Arm,
217 pick: (cell: CellSummary) => number
218 ): number | undefined => {
219 const chosen = cells.filter(cell => cell.split === split && cell.arm === arm && cell.n > 0);
220 return chosen.length === 0
221 ? undefined
222 : chosen.reduce((sum, cell) => sum + pick(cell), 0) / chosen.length;
223 };
224
225 export const renderSummary = (info: BatchInfo, runs: ReadonlyArray<RunRecord>): string => {
226 const cells = summarize(runs);
227 const totalCost = runs
228 .filter(run => run.reusedFrom === undefined)
229 .reduce((sum, run) => sum + run.costUsd, 0);
230 const lines = [
231 `# Batch ${info.id}`,
232 '',
233 `${info.agent} ${info.agentVersion}, ${info.model}, ${info.runs} run(s) per journey and arm.`,
234 `Revision ${info.revision}${info.dirty ? ' with uncommitted changes' : ''}; skill ${info.skillSha256.slice(0, 12)}; harness ${info.harnessSha256.slice(0, 12)}.`,
235 `Spent ${money(totalCost)} on new runs; ${runs.filter(run => run.reusedFrom !== undefined).length} run(s) reused.`,
236 '',
237 '| split | journey | arm | passed | pass@1 (95% CI) | pass@k | pass^k | wrong tree | mean cost | infra |',
238 '|---|---|---|---|---|---|---|---|---|---|',
239 ...cells.map(
240 cell =>
241 `| ${cell.split} | ${cell.journey} | ${cell.arm} | ${cell.passed}/${cell.n} | ${percent(cell.passAt1)} (${percent(cell.interval.low)}–${percent(cell.interval.high)}) | ${percent(cell.passAtK)} | ${percent(cell.passHatK)} | ${percent(cell.wrongTreeRate)} | ${money(cell.meanCostUsd)} | ${cell.infra} |`
242 ),
243 '',
244 '| split | arm | mean pass@1 | mean pass^k | mean wrong tree |',
245 '|---|---|---|---|---|',
246 ];
247 for (const split of ['train', 'holdout'] as const) {
248 for (const arm of info.arms) {
249 const passAt1 = splitMean(cells, split, arm, cell => cell.passAt1);
250 if (passAt1 === undefined) continue;
251 lines.push(
252 `| ${split} | ${arm} | ${percent(passAt1)} | ${percent(splitMean(cells, split, arm, cell => cell.passHatK) ?? 0)} | ${percent(splitMean(cells, split, arm, cell => cell.wrongTreeRate) ?? 0)} |`
253 );
254 }
255 }
256 const failures = cells.filter(cell => Object.keys(cell.failedChecks).length > 0);
257 if (failures.length > 0) {
258 lines.push('', '## Failed checks', '');
259 for (const cell of failures) {
260 lines.push(`- ${cell.journey} (${cell.arm}):`);
261 for (const [name, count] of Object.entries(cell.failedChecks)) {
262 lines.push(` - ${count}/${cell.n} ${name}`);
263 }
264 }
265 }
266 return `${lines.join('\n')}\n`;
267 };
268
269 /**
270 * Baseline against candidate, per journey and per split, on the skill arm.
271 * Refuses batches graded by different harnesses: their numbers do not compare.
272 */
273 export const renderComparison = (
274 baseline: { info: BatchInfo; runs: ReadonlyArray<RunRecord> },
275 candidate: { info: BatchInfo; runs: ReadonlyArray<RunRecord> },
276 options: { arm: Arm; allowHarnessChange: boolean }
277 ): string => {
278 const problems: string[] = [];
279 if (baseline.info.harnessSha256 !== candidate.info.harnessSha256) {
280 problems.push(
281 'The batches were graded by different harnesses (journeys, graders, person or fake backend changed).'
282 );
283 }
284 if (baseline.info.model !== candidate.info.model) {
285 problems.push(`Different models: ${baseline.info.model} and ${candidate.info.model}.`);
286 }
287 if (baseline.info.agent !== candidate.info.agent) {
288 problems.push(`Different agents: ${baseline.info.agent} and ${candidate.info.agent}.`);
289 }
290 if (problems.length > 0 && !options.allowHarnessChange) {
291 throw new Error(`${problems.join('\n')}\nPass --allow-harness-change to compare anyway.`);
292 }
293
294 const before = new Map(
295 summarize(baseline.runs)
296 .filter(cell => cell.arm === options.arm)
297 .map(cell => [cell.journey, cell])
298 );
299 const after = summarize(candidate.runs).filter(cell => cell.arm === options.arm);
300 const signed = (value: number) => `${value > 0 ? '+' : ''}${Math.round(value * 100)}`;
301 const lines = [
302 `# ${baseline.info.id} → ${candidate.info.id} (${options.arm} arm)`,
303 '',
304 ...problems.map(problem => `> ${problem}`),
305 ...(problems.length > 0 ? [''] : []),
306 `Skill ${baseline.info.skillSha256.slice(0, 12)} → ${candidate.info.skillSha256.slice(0, 12)}.`,
307 '',
308 '| split | journey | pass@1 | Δ pass@1 | pass^k | Δ pass^k | wrong tree | Δ wrong tree |',
309 '|---|---|---|---|---|---|---|---|',
310 ];
311 const deltas = { train: [] as number[][], holdout: [] as number[][] };
312 for (const cell of after) {
313 const old = before.get(cell.journey);
314 if (!old || old.n === 0 || cell.n === 0) continue;
315 const delta = [
316 cell.passAt1 - old.passAt1,
317 cell.passHatK - old.passHatK,
318 cell.wrongTreeRate - old.wrongTreeRate,
319 ];
320 deltas[cell.split].push(delta);
321 lines.push(
322 `| ${cell.split} | ${cell.journey} | ${percent(old.passAt1)} → ${percent(cell.passAt1)} | ${signed(delta[0]!)} | ${percent(old.passHatK)} → ${percent(cell.passHatK)} | ${signed(delta[1]!)} | ${percent(old.wrongTreeRate)} → ${percent(cell.wrongTreeRate)} | ${signed(delta[2]!)} |`
323 );
324 }
325 lines.push(
326 '',
327 '| split | journeys | mean Δ pass@1 | mean Δ pass^k | mean Δ wrong tree |',
328 '|---|---|---|---|---|'
329 );
330 for (const split of ['train', 'holdout'] as const) {
331 const rows = deltas[split];
332 if (rows.length === 0) continue;
333 const mean = (index: number) => rows.reduce((sum, row) => sum + row[index]!, 0) / rows.length;
334 lines.push(
335 `| ${split} | ${rows.length} | ${signed(mean(0))} | ${signed(mean(1))} | ${signed(mean(2))} |`
336 );
337 }
338 return `${lines.join('\n')}\n`;
339 };