Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 449 lines 16 KB
Raw
1 /**
2 * Live, repeated journey runs: the measurement a skill change is judged by.
3 *
4 * bun run eval.ts run --model claude-haiku-5-5 [options]
5 * bun run eval.ts report <batch>
6 * bun run eval.ts compare <baseline batch> <candidate batch> [--arm skill]
7 * bun run eval.ts keep <batch> <decision-name>
8 *
9 * A batch is named by its id under `results/`, or by the path of a batch
10 * directory kept in the repository (see `evals/`).
11 *
12 * `run` options:
13 * --agent claude-code|loop Claude Code with the skill installed (default), or
14 * the minimal loop with the skill as system prompt
15 * --runs <k> runs per journey and arm (default 3)
16 * --arms skill,no-skill which arms to run (default skill)
17 * --split train|holdout|all which journeys (default all)
18 * --journeys <id,id> exactly these journeys
19 * --max-cost-usd <usd> stop starting runs once this much is spent (default 2)
20 * --run-budget-usd <usd> Claude Code stops a single run at this cost (default 0.5)
21 * --concurrency <n> runs at once (default 2)
22 * --label <name> appended to the batch id
23 * --force run everything, reusing no earlier run
24 * --dry print the plan; start nothing, spend nothing
25 * --smoke one journey, one run, to check the setup
26 *
27 * The API key is read from ANTHROPIC_API_KEY and stays in this process: the
28 * agent reaches the model through `src/model-proxy.ts`. Results go to
29 * `results/<batch>/` (git-ignored). See HILLCLIMB.md for how to use them.
30 */
31
32 import { execFileSync } from 'node:child_process';
33 import { createHash } from 'node:crypto';
34 import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
35 import { join, relative, resolve } from 'node:path';
36 import { parseArgs } from 'node:util';
37 import { startJourneyContainer } from '@e2e-tests/utils';
38 import { z } from 'zod';
39 import { anthropicAdapter } from './src/anthropic-adapter';
40 import {
41 AGENTS,
42 ARMS,
43 fingerprint,
44 previousRuns,
45 readBatch,
46 renderComparison,
47 renderSummary,
48 treeHash,
49 type AgentName,
50 type Arm,
51 type BatchInfo,
52 type RunRecord,
53 } from './src/batch';
54 import { CLAUDE_CODE_VERSION, claudeCodeDriver, ensureClaudeCodeBinary } from './src/claude-code';
55 import { MAX_OUTPUT_TOKENS } from './src/adapters';
56 import { JOURNEYS, type Journey, type JourneyRun } from './src/journeys';
57 import { ALLOWED_MODELS, startModelProxy, type ModelProxy } from './src/model-proxy';
58 import { loopDriver, runJourneyWith, type Driver } from './src/run-journey';
59 import { buildSkill } from './src/skill';
60
61 const HERE = import.meta.dir;
62 const RESULTS = join(HERE, 'results');
63 const REPO_TS = resolve(HERE, '..', '..', '..');
64 /**
65 * What drives and grades a run: the journeys and their checks, the person,
66 * the agents, and the fake backend. A change here makes earlier results
67 * incomparable. The reporting code (`batch.ts`, `metrics.ts`) is left out.
68 */
69 const HARNESS_PATHS = [
70 'e2e-tests/cli/fake-backend/src',
71 ...[
72 'agent.ts',
73 'anthropic-adapter.ts',
74 'claude-code.ts',
75 'journeys.ts',
76 'person.ts',
77 'run-journey.ts',
78 'scrub.ts',
79 'skill.ts',
80 ].map(file => `e2e-tests/cli/agent-journeys/src/${file}`),
81 ];
82 /** Wall-clock limit for one run's agent. */
83 const RUN_DEADLINE_MS = 6 * 60_000;
84 /** Shell commands one run may start. The longest recorded run used eleven. */
85 const MAX_COMMANDS = 40;
86
87 const fail = (message: string): never => {
88 console.error(message);
89 process.exit(2);
90 };
91
92 const git = (...args: string[]): string =>
93 execFileSync('git', args, { cwd: HERE, encoding: 'utf8' }).trim();
94
95 /** A batch id under `results/`, or the path of a batch directory kept elsewhere (`evals/`). */
96 const batchDirectory = (name: string): string =>
97 existsSync(join(name, 'batch.json')) ? resolve(name) : resolve(RESULTS, name);
98
99 const [command, ...rest] = process.argv.slice(2);
100
101 if (command === 'report') {
102 const [batch] = rest;
103 if (!batch) fail('Usage: bun run eval.ts report <batch>');
104 const { info, runs } = readBatch(batchDirectory(batch!));
105 process.stdout.write(renderSummary(info, runs));
106 process.exit(0);
107 }
108
109 if (command === 'keep') {
110 const [batch, decision] = rest;
111 if (!batch || !decision || !/^[a-z0-9][a-z0-9-]*$/.test(decision)) {
112 fail('Usage: bun run eval.ts keep <batch> <decision-name>');
113 }
114 const source = batchDirectory(batch!);
115 const { info } = readBatch(source);
116 const target = join(HERE, 'evals', decision!, info.id);
117 mkdirSync(target, { recursive: true });
118 // The records and the summary, not the logs: they are large and hold every command's output.
119 for (const name of ['batch.json', 'runs.jsonl', 'summary.md']) {
120 const text = readFileSync(join(source, name), 'utf8');
121 if (/sk-ant-|sk-journey-/.test(text)) fail(`${name} holds a key or a proxy token; not kept.`);
122 writeFileSync(join(target, name), text);
123 }
124 console.log(`kept ${relative(process.cwd(), target)}`);
125 process.exit(0);
126 }
127
128 if (command === 'compare') {
129 const { values, positionals } = parseArgs({
130 args: rest,
131 allowPositionals: true,
132 options: {
133 arm: { type: 'string', default: 'skill' },
134 'allow-harness-change': { type: 'boolean', default: false },
135 },
136 });
137 const [baseline, candidate] = positionals;
138 const arm = z.enum(ARMS).safeParse(values.arm);
139 if (!baseline || !candidate || !arm.success) {
140 fail(
141 'Usage: bun run eval.ts compare <baseline batch> <candidate batch> [--arm skill|no-skill]'
142 );
143 }
144 process.stdout.write(
145 renderComparison(readBatch(batchDirectory(baseline!)), readBatch(batchDirectory(candidate!)), {
146 arm: arm.data!,
147 allowHarnessChange: values['allow-harness-change'] ?? false,
148 })
149 );
150 process.exit(0);
151 }
152
153 if (command !== 'run') {
154 fail('Usage: bun run eval.ts run|report|compare|keep … (see the header of eval.ts)');
155 }
156
157 const { values } = parseArgs({
158 args: rest,
159 options: {
160 model: { type: 'string' },
161 agent: { type: 'string', default: 'claude-code' },
162 runs: { type: 'string', default: '3' },
163 arms: { type: 'string', default: 'skill' },
164 split: { type: 'string', default: 'all' },
165 journeys: { type: 'string' },
166 'max-cost-usd': { type: 'string', default: '2' },
167 'run-budget-usd': { type: 'string', default: '0.5' },
168 concurrency: { type: 'string', default: '2' },
169 label: { type: 'string' },
170 force: { type: 'boolean', default: false },
171 dry: { type: 'boolean', default: false },
172 smoke: { type: 'boolean', default: false },
173 },
174 });
175
176 const Options = z.object({
177 model: z.enum(ALLOWED_MODELS),
178 agent: z.enum(AGENTS),
179 runs: z.coerce.number().int().min(1).max(20),
180 arms: z.array(z.enum(ARMS)).min(1),
181 split: z.enum(['train', 'holdout', 'all']),
182 maxCostUsd: z.coerce.number().positive(),
183 runBudgetUsd: z.coerce.number().positive(),
184 concurrency: z.coerce.number().int().min(1).max(8),
185 label: z
186 .string()
187 .regex(/^[a-z0-9][a-z0-9-]*$/)
188 .optional(),
189 });
190 const parsed = Options.safeParse({
191 model: values.model,
192 agent: values.agent,
193 runs: values.smoke ? '1' : values.runs,
194 arms: values.arms?.split(',').map(arm => arm.trim()),
195 split: values.split,
196 maxCostUsd: values['max-cost-usd'],
197 runBudgetUsd: values['run-budget-usd'],
198 concurrency: values.concurrency,
199 label: values.label,
200 });
201 if (!parsed.success) {
202 fail(`${z.prettifyError(parsed.error)}\n\nModels: ${ALLOWED_MODELS.join(', ')}`);
203 }
204 const options = parsed.data!;
205
206 const named = values.journeys?.split(',').map(id => id.trim());
207 const unknown = named?.filter(id => !JOURNEYS.some(journey => journey.id === id)) ?? [];
208 if (unknown.length > 0) fail(`Unknown journeys: ${unknown.join(', ')}`);
209 let journeys: ReadonlyArray<Journey> = JOURNEYS.filter(
210 journey =>
211 (named === undefined || named.includes(journey.id)) &&
212 (options.split === 'all' || journey.split === options.split)
213 );
214 if (values.smoke) journeys = journeys.slice(0, 1);
215 if (journeys.length === 0) fail('No journey matches.');
216
217 const apiKey = process.env.ANTHROPIC_API_KEY;
218 if (!apiKey && !values.dry) fail('ANTHROPIC_API_KEY is not set; nothing was started.');
219
220 // What the batch measures. The CLI is the binary a journey container would
221 // run now: starting one builds the image when the checkout changed. The
222 // binary is hashed, not the image, which also holds the test sources.
223 const probe = await startJourneyContainer({ label: 'eval-probe' });
224 const binaries = await probe.exec('cd /usr/local/bin && sha256sum composio *.mjs | sha256sum', {
225 cwd: '/',
226 });
227 await probe.stop();
228 if (binaries.exitCode !== 0) fail(`Could not hash the CLI binary: ${binaries.stderr}`);
229 const cliSha256 = binaries.stdout.split(' ')[0]!.trim();
230
231 const agentVersion = options.agent === 'claude-code' ? CLAUDE_CODE_VERSION : 'loop';
232 const skillSha256 = createHash('sha256').update(JSON.stringify(buildSkill())).digest('hex');
233 const harnessSha256 = treeHash(REPO_TS, HARNESS_PATHS);
234 const startedAt = new Date();
235 const batchId = `${startedAt.toISOString().replace(/[:.]/g, '-').slice(0, 19)}-${options.agent}-${options.model}${options.label ? `-${options.label}` : ''}`;
236 const info: BatchInfo = {
237 id: batchId,
238 startedAt: startedAt.toISOString(),
239 agent: options.agent,
240 agentVersion,
241 model: options.model,
242 runs: options.runs,
243 arms: options.arms,
244 journeys: journeys.map(journey => journey.id),
245 revision: git('rev-parse', '--short', 'HEAD'),
246 dirty: git('status', '--porcelain', '--', ':/ts/packages/cli', ':/ts/e2e-tests/cli') !== '',
247 skillSha256,
248 harnessSha256,
249 cliSha256,
250 maxCostUsd: options.maxCostUsd,
251 };
252
253 interface Planned {
254 readonly journey: Journey;
255 readonly arm: Arm;
256 readonly attempt: number;
257 readonly fingerprint: string;
258 }
259
260 const earlier = values.force ? new Map<string, RunRecord[]>() : previousRuns(RESULTS);
261 const reused: RunRecord[] = [];
262 const planned: Planned[] = [];
263 for (const journey of journeys) {
264 for (const arm of options.arms) {
265 const print = fingerprint({
266 journey: journey.id,
267 arm,
268 agent: options.agent,
269 agentVersion,
270 model: options.model,
271 skillSha256,
272 harnessSha256,
273 cliSha256,
274 });
275 const available = (earlier.get(print) ?? []).slice(-options.runs);
276 reused.push(...available.map((run, index) => ({ ...run, attempt: index + 1 })));
277 for (let attempt = available.length + 1; attempt <= options.runs; attempt += 1) {
278 planned.push({ journey, arm, attempt, fingerprint: print });
279 }
280 }
281 }
282
283 console.log(`batch ${batchId}`);
284 console.log(`agent ${options.agent} ${agentVersion}, ${options.model}`);
285 console.log(`journeys ${journeys.map(journey => `${journey.id} (${journey.split})`).join(', ')}`);
286 console.log(`arms ${options.arms.join(', ')}; ${options.runs} run(s) each`);
287 console.log(
288 `runs ${planned.length} new, ${reused.length} reused; budget $${options.maxCostUsd}`
289 );
290 if (values.dry) {
291 for (const run of planned)
292 console.log(` would run ${run.journey.id} ${run.arm} #${run.attempt}`);
293 process.exit(0);
294 }
295
296 const directory = join(RESULTS, batchId);
297 mkdirSync(join(directory, 'logs'), { recursive: true });
298 writeFileSync(join(directory, 'batch.json'), `${JSON.stringify(info, null, 2)}\n`);
299 const record = (run: RunRecord) =>
300 appendFileSync(join(directory, 'runs.jsonl'), `${JSON.stringify(run)}\n`);
301 for (const run of reused) record(run);
302
303 const proxy: ModelProxy = await startModelProxy({ apiKey: apiKey! });
304 const binary = options.agent === 'claude-code' ? ensureClaudeCodeBinary() : undefined;
305 let spent = 0;
306 let interrupted = false;
307 process.on('SIGINT', () => {
308 interrupted = true;
309 console.log('\ninterrupted: finishing the runs in flight, starting no more');
310 });
311
312 /** Nothing that could carry a credential is written. */
313 const redact = (text: string): string => {
314 const cleaned = text.replaceAll(proxy.token, '<proxy-token>');
315 if (cleaned.includes(apiKey!) || /sk-ant-/.test(cleaned)) {
316 throw new Error('Refusing to write a log that contains an API key.');
317 }
318 return cleaned;
319 };
320
321 const driverFor = (arm: Arm, budgetUsd: number, onEvent: (event: unknown) => void): Driver =>
322 options.agent === 'claude-code'
323 ? claudeCodeDriver({
324 model: options.model,
325 proxy,
326 binary: binary!,
327 withSkill: arm === 'skill',
328 budgetUsd,
329 deadlineMs: RUN_DEADLINE_MS,
330 maxCommands: MAX_COMMANDS,
331 onEvent,
332 })
333 : loopDriver(
334 inputs =>
335 anthropicAdapter({
336 baseURL: 'https://api.anthropic.com',
337 apiKey: apiKey!,
338 model: options.model,
339 maxTokens: MAX_OUTPUT_TOKENS,
340 ...inputs,
341 }),
342 { skill: arm === 'skill' ? 'system-prompt' : 'none', scrubbed: false }
343 );
344
345 const runOne = async (plan: Planned): Promise<void> => {
346 const name = `${plan.journey.id}.${plan.arm}.${plan.attempt}`;
347 const remaining = options.maxCostUsd - spent;
348 if (interrupted || remaining <= 0) {
349 console.log(` skipped ${name}: ${interrupted ? 'interrupted' : 'budget spent'}`);
350 return;
351 }
352 const events: unknown[] = [];
353 const started = Date.now();
354 const refusedBefore = proxy.refused().length;
355 let run: JourneyRun | undefined;
356 try {
357 run = await runJourneyWith(
358 plan.journey,
359 driverFor(plan.arm, Math.min(options.runBudgetUsd, remaining), event => events.push(event))
360 );
361 const checks = plan.journey.checks(run);
362 const detours = (plan.journey.detours?.(run) ?? []).filter(detour => !detour.ok);
363 const stop =
364 typeof run.transcript.stop === 'string' ? run.transcript.stop : run.transcript.stop.provider;
365 // A harness or provider failure says nothing about the agent.
366 const refused = proxy.refused().slice(refusedBefore);
367 const infra = stop.startsWith('exited ') || refused.length > 0;
368 const costUsd = run.usage?.costUsd ?? 0;
369 spent += costUsd;
370 const result: RunRecord = {
371 journey: plan.journey.id,
372 split: plan.journey.split,
373 arm: plan.arm,
374 attempt: plan.attempt,
375 fingerprint: plan.fingerprint,
376 outcome: infra ? 'infra' : checks.every(check => check.ok) ? 'passed' : 'failed',
377 checks: checks.map(check => ({
378 name: check.name,
379 ok: check.ok,
380 ...(check.ok ? {} : { seen: JSON.stringify(check.seen ?? null).slice(0, 500) }),
381 })),
382 detours: detours.map(detour => ({
383 name: detour.name,
384 ok: false,
385 seen: JSON.stringify(detour.seen ?? null).slice(0, 500),
386 })),
387 stop,
388 modelTurns: run.transcript.modelTurns,
389 commands: run.transcript.commands.map(item => ({
390 command: item.command,
391 exitCode: item.exitCode,
392 })),
393 costUsd,
394 durationMs: Date.now() - started,
395 ...(infra ? { error: `stop: ${stop}; proxy refused: ${refused.join('; ')}` } : {}),
396 };
397 writeFileSync(
398 join(directory, 'logs', `${name}.json`),
399 redact(
400 JSON.stringify(
401 { ...result, transcript: run.transcript, requests: run.requests, events },
402 null,
403 2
404 )
405 )
406 );
407 record(result);
408 console.log(
409 ` ${result.outcome.padEnd(6)} ${name} $${costUsd.toFixed(3)} ${result.commands.length} cmd ${detours.length} detour(s)`
410 );
411 } catch (error) {
412 const message = error instanceof Error ? error.message : String(error);
413 record({
414 journey: plan.journey.id,
415 split: plan.journey.split,
416 arm: plan.arm,
417 attempt: plan.attempt,
418 fingerprint: plan.fingerprint,
419 outcome: 'infra',
420 checks: [],
421 detours: [],
422 stop: 'error',
423 modelTurns: 0,
424 commands: [],
425 costUsd: 0,
426 durationMs: Date.now() - started,
427 error: redact(message).slice(0, 1000),
428 });
429 console.log(` infra ${name} ${message.slice(0, 200)}`);
430 } finally {
431 await run?.rig.stop();
432 }
433 };
434
435 try {
436 const queue = [...planned];
437 await Promise.all(
438 Array.from({ length: Math.min(options.concurrency, queue.length) }, async () => {
439 for (let next = queue.shift(); next !== undefined; next = queue.shift()) await runOne(next);
440 })
441 );
442 } finally {
443 await proxy.close();
444 }
445
446 const summary = renderSummary(info, readBatch(directory).runs);
447 writeFileSync(join(directory, 'summary.md'), summary);
448 process.stdout.write(`\n${summary}`);
449 console.log(`results ${directory}`);