Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | /** |
| 2 | * Batches of live runs: what each run recorded, how runs are reused, and the |
| 3 | * summaries `eval.ts` prints. |
| 4 | * |
| 5 | * A batch is a directory under `results/`: `batch.json` says what was |
| 6 | * measured, `runs.jsonl` has one line per run, `logs/` the full transcripts. |
| 7 | */ |
| 8 | |
| 9 | import { createHash } from 'node:crypto'; |
| 10 | import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; |
| 11 | import { join, relative } from 'node:path'; |
| 12 | import { z } from 'zod'; |
| 13 | import { passAtK, passHatK, wilson } from './metrics'; |
| 14 | |
| 15 | export const ARMS = ['skill', 'no-skill'] as const; |
| 16 | export type Arm = (typeof ARMS)[number]; |
| 17 | |
| 18 | export const AGENTS = ['claude-code', 'loop'] as const; |
| 19 | export type AgentName = (typeof AGENTS)[number]; |
| 20 | |
| 21 | const CheckRecord = z.object({ name: z.string(), ok: z.boolean(), seen: z.string().optional() }); |
| 22 | |
| 23 | export const RunRecord = z.object({ |
| 24 | journey: z.string(), |
| 25 | split: z.enum(['train', 'holdout']), |
| 26 | arm: z.enum(ARMS), |
| 27 | attempt: z.number(), |
| 28 | fingerprint: z.string(), |
| 29 | /** |
| 30 | * `passed` and `failed` are graded. `infra` means the run says nothing about |
| 31 | * the agent (the harness or the provider failed) and is left out of every rate. |
| 32 | */ |
| 33 | outcome: z.enum(['passed', 'failed', 'infra']), |
| 34 | checks: z.array(CheckRecord), |
| 35 | /** Detours taken; never a failure, see `Journey.detours`. */ |
| 36 | detours: z.array(CheckRecord), |
| 37 | stop: z.string(), |
| 38 | modelTurns: z.number(), |
| 39 | commands: z.array(z.object({ command: z.string(), exitCode: z.number() })), |
| 40 | costUsd: z.number(), |
| 41 | durationMs: z.number(), |
| 42 | error: z.string().optional(), |
| 43 | /** Set when the run was copied from an earlier batch with the same fingerprint. */ |
| 44 | reusedFrom: z.string().optional(), |
| 45 | }); |
| 46 | export type RunRecord = z.infer<typeof RunRecord>; |
| 47 | |
| 48 | export const BatchInfo = z.object({ |
| 49 | id: z.string(), |
| 50 | startedAt: z.string(), |
| 51 | agent: z.enum(AGENTS), |
| 52 | agentVersion: z.string(), |
| 53 | model: z.string(), |
| 54 | runs: z.number(), |
| 55 | arms: z.array(z.enum(ARMS)), |
| 56 | journeys: z.array(z.string()), |
| 57 | revision: z.string(), |
| 58 | /** Uncommitted changes under `ts/packages/cli` or the harness. */ |
| 59 | dirty: z.boolean(), |
| 60 | skillSha256: z.string(), |
| 61 | harnessSha256: z.string(), |
| 62 | /** sha256 of the CLI binary and its companion modules, as the journey container has them. */ |
| 63 | cliSha256: z.string(), |
| 64 | maxCostUsd: z.number(), |
| 65 | }); |
| 66 | export type BatchInfo = z.infer<typeof BatchInfo>; |
| 67 | |
| 68 | /** sha256 of the given files and of every file under the given directories, by relative path. */ |
| 69 | export const treeHash = (root: string, paths: ReadonlyArray<string>): string => { |
| 70 | const hash = createHash('sha256'); |
| 71 | const walk = (path: string) => { |
| 72 | if (statSync(path).isDirectory()) { |
| 73 | for (const name of readdirSync(path).sort()) { |
| 74 | if (name !== 'node_modules') walk(join(path, name)); |
| 75 | } |
| 76 | return; |
| 77 | } |
| 78 | hash |
| 79 | .update(`${relative(root, path)}\0`) |
| 80 | .update(readFileSync(path)) |
| 81 | .update('\0'); |
| 82 | }; |
| 83 | for (const path of paths) walk(join(root, path)); |
| 84 | return hash.digest('hex'); |
| 85 | }; |
| 86 | |
| 87 | /** |
| 88 | * What makes two runs interchangeable: the same journey, under the same |
| 89 | * graders, with the same CLI, agent, model, and (for the skill arm) skill. |
| 90 | */ |
| 91 | export const fingerprint = (parts: { |
| 92 | journey: string; |
| 93 | arm: Arm; |
| 94 | agent: AgentName; |
| 95 | agentVersion: string; |
| 96 | model: string; |
| 97 | skillSha256: string; |
| 98 | harnessSha256: string; |
| 99 | cliSha256: string; |
| 100 | }): string => |
| 101 | createHash('sha256') |
| 102 | .update( |
| 103 | JSON.stringify({ |
| 104 | ...parts, |
| 105 | skillSha256: parts.arm === 'skill' ? parts.skillSha256 : 'none', |
| 106 | }) |
| 107 | ) |
| 108 | .digest('hex') |
| 109 | .slice(0, 16); |
| 110 | |
| 111 | export const readRuns = (batchDirectory: string): RunRecord[] => { |
| 112 | const path = join(batchDirectory, 'runs.jsonl'); |
| 113 | if (!existsSync(path)) return []; |
| 114 | return readFileSync(path, 'utf8') |
| 115 | .split('\n') |
| 116 | .filter(line => line.trim() !== '') |
| 117 | .map(line => RunRecord.parse(JSON.parse(line))); |
| 118 | }; |
| 119 | |
| 120 | export const readBatch = (batchDirectory: string): { info: BatchInfo; runs: RunRecord[] } => ({ |
| 121 | info: BatchInfo.parse(JSON.parse(readFileSync(join(batchDirectory, 'batch.json'), 'utf8'))), |
| 122 | runs: readRuns(batchDirectory), |
| 123 | }); |
| 124 | |
| 125 | /** Graded runs from earlier batches, by fingerprint, oldest first. */ |
| 126 | export const previousRuns = (resultsDirectory: string): Map<string, RunRecord[]> => { |
| 127 | const byFingerprint = new Map<string, RunRecord[]>(); |
| 128 | if (!existsSync(resultsDirectory)) return byFingerprint; |
| 129 | for (const name of readdirSync(resultsDirectory).sort()) { |
| 130 | const directory = join(resultsDirectory, name); |
| 131 | if (!statSync(directory).isDirectory()) continue; |
| 132 | for (const run of readRuns(directory)) { |
| 133 | if (run.outcome === 'infra' || run.reusedFrom !== undefined) continue; |
| 134 | const list = byFingerprint.get(run.fingerprint) ?? []; |
| 135 | list.push({ ...run, reusedFrom: name }); |
| 136 | byFingerprint.set(run.fingerprint, list); |
| 137 | } |
| 138 | } |
| 139 | return byFingerprint; |
| 140 | }; |
| 141 | |
| 142 | export interface CellSummary { |
| 143 | readonly journey: string; |
| 144 | readonly split: 'train' | 'holdout'; |
| 145 | readonly arm: Arm; |
| 146 | /** Graded runs. */ |
| 147 | readonly n: number; |
| 148 | readonly passed: number; |
| 149 | readonly infra: number; |
| 150 | readonly passAt1: number; |
| 151 | readonly interval: { low: number; high: number }; |
| 152 | /** At k = the number of graded runs. */ |
| 153 | readonly passAtK: number; |
| 154 | readonly passHatK: number; |
| 155 | /** Share of graded runs that took at least one detour. */ |
| 156 | readonly wrongTreeRate: number; |
| 157 | readonly meanCostUsd: number; |
| 158 | readonly meanTurns: number; |
| 159 | /** How often each check failed, by name. */ |
| 160 | readonly failedChecks: Readonly<Record<string, number>>; |
| 161 | } |
| 162 | |
| 163 | export const summarize = (runs: ReadonlyArray<RunRecord>): CellSummary[] => { |
| 164 | const cells = new Map<string, RunRecord[]>(); |
| 165 | for (const run of runs) { |
| 166 | const key = `${run.journey}\0${run.arm}`; |
| 167 | cells.set(key, [...(cells.get(key) ?? []), run]); |
| 168 | } |
| 169 | return [...cells.values()] |
| 170 | .map(cell => { |
| 171 | const [first] = cell; |
| 172 | const graded = cell.filter(run => run.outcome !== 'infra'); |
| 173 | const n = graded.length; |
| 174 | const passed = graded.filter(run => run.outcome === 'passed').length; |
| 175 | const failedChecks: Record<string, number> = {}; |
| 176 | for (const run of graded) { |
| 177 | for (const check of run.checks.filter(item => !item.ok)) { |
| 178 | failedChecks[check.name] = (failedChecks[check.name] ?? 0) + 1; |
| 179 | } |
| 180 | } |
| 181 | const mean = (values: number[]) => |
| 182 | values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length; |
| 183 | return { |
| 184 | journey: first!.journey, |
| 185 | split: first!.split, |
| 186 | arm: first!.arm, |
| 187 | n, |
| 188 | passed, |
| 189 | infra: cell.length - n, |
| 190 | passAt1: n === 0 ? 0 : passed / n, |
| 191 | interval: wilson(passed, n), |
| 192 | passAtK: n === 0 ? 0 : passAtK(n, passed, n), |
| 193 | passHatK: n === 0 ? 0 : passHatK(n, passed, n), |
| 194 | wrongTreeRate: n === 0 ? 0 : graded.filter(run => run.detours.length > 0).length / n, |
| 195 | meanCostUsd: mean(cell.map(run => run.costUsd)), |
| 196 | meanTurns: mean(graded.map(run => run.modelTurns)), |
| 197 | failedChecks, |
| 198 | }; |
| 199 | }) |
| 200 | .sort((left, right) => |
| 201 | left.split === right.split |
| 202 | ? left.journey.localeCompare(right.journey) || left.arm.localeCompare(right.arm) |
| 203 | : left.split === 'train' |
| 204 | ? -1 |
| 205 | : 1 |
| 206 | ); |
| 207 | }; |
| 208 | |
| 209 | const percent = (value: number) => `${Math.round(value * 100)}%`; |
| 210 | const money = (value: number) => `$${value.toFixed(value < 1 ? 3 : 2)}`; |
| 211 | |
| 212 | /** Mean over journeys, so a journey with more runs does not count for more. */ |
| 213 | const splitMean = ( |
| 214 | cells: ReadonlyArray<CellSummary>, |
| 215 | split: 'train' | 'holdout', |
| 216 | arm: Arm, |
| 217 | pick: (cell: CellSummary) => number |
| 218 | ): number | undefined => { |
| 219 | const chosen = cells.filter(cell => cell.split === split && cell.arm === arm && cell.n > 0); |
| 220 | return chosen.length === 0 |
| 221 | ? undefined |
| 222 | : chosen.reduce((sum, cell) => sum + pick(cell), 0) / chosen.length; |
| 223 | }; |
| 224 | |
| 225 | export const renderSummary = (info: BatchInfo, runs: ReadonlyArray<RunRecord>): string => { |
| 226 | const cells = summarize(runs); |
| 227 | const totalCost = runs |
| 228 | .filter(run => run.reusedFrom === undefined) |
| 229 | .reduce((sum, run) => sum + run.costUsd, 0); |
| 230 | const lines = [ |
| 231 | `# Batch ${info.id}`, |
| 232 | '', |
| 233 | `${info.agent} ${info.agentVersion}, ${info.model}, ${info.runs} run(s) per journey and arm.`, |
| 234 | `Revision ${info.revision}${info.dirty ? ' with uncommitted changes' : ''}; skill ${info.skillSha256.slice(0, 12)}; harness ${info.harnessSha256.slice(0, 12)}.`, |
| 235 | `Spent ${money(totalCost)} on new runs; ${runs.filter(run => run.reusedFrom !== undefined).length} run(s) reused.`, |
| 236 | '', |
| 237 | '| split | journey | arm | passed | pass@1 (95% CI) | pass@k | pass^k | wrong tree | mean cost | infra |', |
| 238 | '|---|---|---|---|---|---|---|---|---|---|', |
| 239 | ...cells.map( |
| 240 | cell => |
| 241 | `| ${cell.split} | ${cell.journey} | ${cell.arm} | ${cell.passed}/${cell.n} | ${percent(cell.passAt1)} (${percent(cell.interval.low)}–${percent(cell.interval.high)}) | ${percent(cell.passAtK)} | ${percent(cell.passHatK)} | ${percent(cell.wrongTreeRate)} | ${money(cell.meanCostUsd)} | ${cell.infra} |` |
| 242 | ), |
| 243 | '', |
| 244 | '| split | arm | mean pass@1 | mean pass^k | mean wrong tree |', |
| 245 | '|---|---|---|---|---|', |
| 246 | ]; |
| 247 | for (const split of ['train', 'holdout'] as const) { |
| 248 | for (const arm of info.arms) { |
| 249 | const passAt1 = splitMean(cells, split, arm, cell => cell.passAt1); |
| 250 | if (passAt1 === undefined) continue; |
| 251 | lines.push( |
| 252 | `| ${split} | ${arm} | ${percent(passAt1)} | ${percent(splitMean(cells, split, arm, cell => cell.passHatK) ?? 0)} | ${percent(splitMean(cells, split, arm, cell => cell.wrongTreeRate) ?? 0)} |` |
| 253 | ); |
| 254 | } |
| 255 | } |
| 256 | const failures = cells.filter(cell => Object.keys(cell.failedChecks).length > 0); |
| 257 | if (failures.length > 0) { |
| 258 | lines.push('', '## Failed checks', ''); |
| 259 | for (const cell of failures) { |
| 260 | lines.push(`- ${cell.journey} (${cell.arm}):`); |
| 261 | for (const [name, count] of Object.entries(cell.failedChecks)) { |
| 262 | lines.push(` - ${count}/${cell.n} ${name}`); |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | return `${lines.join('\n')}\n`; |
| 267 | }; |
| 268 | |
| 269 | /** |
| 270 | * Baseline against candidate, per journey and per split, on the skill arm. |
| 271 | * Refuses batches graded by different harnesses: their numbers do not compare. |
| 272 | */ |
| 273 | export const renderComparison = ( |
| 274 | baseline: { info: BatchInfo; runs: ReadonlyArray<RunRecord> }, |
| 275 | candidate: { info: BatchInfo; runs: ReadonlyArray<RunRecord> }, |
| 276 | options: { arm: Arm; allowHarnessChange: boolean } |
| 277 | ): string => { |
| 278 | const problems: string[] = []; |
| 279 | if (baseline.info.harnessSha256 !== candidate.info.harnessSha256) { |
| 280 | problems.push( |
| 281 | 'The batches were graded by different harnesses (journeys, graders, person or fake backend changed).' |
| 282 | ); |
| 283 | } |
| 284 | if (baseline.info.model !== candidate.info.model) { |
| 285 | problems.push(`Different models: ${baseline.info.model} and ${candidate.info.model}.`); |
| 286 | } |
| 287 | if (baseline.info.agent !== candidate.info.agent) { |
| 288 | problems.push(`Different agents: ${baseline.info.agent} and ${candidate.info.agent}.`); |
| 289 | } |
| 290 | if (problems.length > 0 && !options.allowHarnessChange) { |
| 291 | throw new Error(`${problems.join('\n')}\nPass --allow-harness-change to compare anyway.`); |
| 292 | } |
| 293 | |
| 294 | const before = new Map( |
| 295 | summarize(baseline.runs) |
| 296 | .filter(cell => cell.arm === options.arm) |
| 297 | .map(cell => [cell.journey, cell]) |
| 298 | ); |
| 299 | const after = summarize(candidate.runs).filter(cell => cell.arm === options.arm); |
| 300 | const signed = (value: number) => `${value > 0 ? '+' : ''}${Math.round(value * 100)}`; |
| 301 | const lines = [ |
| 302 | `# ${baseline.info.id} → ${candidate.info.id} (${options.arm} arm)`, |
| 303 | '', |
| 304 | ...problems.map(problem => `> ${problem}`), |
| 305 | ...(problems.length > 0 ? [''] : []), |
| 306 | `Skill ${baseline.info.skillSha256.slice(0, 12)} → ${candidate.info.skillSha256.slice(0, 12)}.`, |
| 307 | '', |
| 308 | '| split | journey | pass@1 | Δ pass@1 | pass^k | Δ pass^k | wrong tree | Δ wrong tree |', |
| 309 | '|---|---|---|---|---|---|---|---|', |
| 310 | ]; |
| 311 | const deltas = { train: [] as number[][], holdout: [] as number[][] }; |
| 312 | for (const cell of after) { |
| 313 | const old = before.get(cell.journey); |
| 314 | if (!old || old.n === 0 || cell.n === 0) continue; |
| 315 | const delta = [ |
| 316 | cell.passAt1 - old.passAt1, |
| 317 | cell.passHatK - old.passHatK, |
| 318 | cell.wrongTreeRate - old.wrongTreeRate, |
| 319 | ]; |
| 320 | deltas[cell.split].push(delta); |
| 321 | lines.push( |
| 322 | `| ${cell.split} | ${cell.journey} | ${percent(old.passAt1)} → ${percent(cell.passAt1)} | ${signed(delta[0]!)} | ${percent(old.passHatK)} → ${percent(cell.passHatK)} | ${signed(delta[1]!)} | ${percent(old.wrongTreeRate)} → ${percent(cell.wrongTreeRate)} | ${signed(delta[2]!)} |` |
| 323 | ); |
| 324 | } |
| 325 | lines.push( |
| 326 | '', |
| 327 | '| split | journeys | mean Δ pass@1 | mean Δ pass^k | mean Δ wrong tree |', |
| 328 | '|---|---|---|---|---|' |
| 329 | ); |
| 330 | for (const split of ['train', 'holdout'] as const) { |
| 331 | const rows = deltas[split]; |
| 332 | if (rows.length === 0) continue; |
| 333 | const mean = (index: number) => rows.reduce((sum, row) => sum + row[index]!, 0) / rows.length; |
| 334 | lines.push( |
| 335 | `| ${split} | ${rows.length} | ${signed(mean(0))} | ${signed(mean(1))} | ${signed(mean(2))} |` |
| 336 | ); |
| 337 | } |
| 338 | return `${lines.join('\n')}\n`; |
| 339 | }; |