1
+/**
2
+ * Live, repeated journey runs: the measurement a skill change is judged by.
3
+ *
4
+ * bun run eval.ts run --model claude-haiku-5-5 [options]
5
+ * bun run eval.ts report <batch>
6
+ * bun run eval.ts compare <baseline batch> <candidate batch> [--arm skill]
7
+ * bun run eval.ts keep <batch> <decision-name>
8
+ *
9
+ * A batch is named by its id under `results/`, or by the path of a batch
10
+ * directory kept in the repository (see `evals/`).
11
+ *
12
+ * `run` options:
13
+ * --agent claude-code|loop Claude Code with the skill installed (default), or
14
+ * the minimal loop with the skill as system prompt
15
+ * --runs <k> runs per journey and arm (default 3)
16
+ * --arms skill,no-skill which arms to run (default skill)
17
+ * --split train|holdout|all which journeys (default all)
18
+ * --journeys <id,id> exactly these journeys
19
+ * --max-cost-usd <usd> stop starting runs once this much is spent (default 2)
20
+ * --run-budget-usd <usd> Claude Code stops a single run at this cost (default 0.5)
21
+ * --concurrency <n> runs at once (default 2)
22
+ * --label <name> appended to the batch id
23
+ * --force run everything, reusing no earlier run
24
+ * --dry print the plan; start nothing, spend nothing
25
+ * --smoke one journey, one run, to check the setup
26
+ *
27
+ * The API key is read from ANTHROPIC_API_KEY and stays in this process: the
28
+ * agent reaches the model through `src/model-proxy.ts`. Results go to
29
+ * `results/<batch>/` (git-ignored). See HILLCLIMB.md for how to use them.
30
+ */
31
+
32
+import { execFileSync } from 'node:child_process';
33
+import { createHash } from 'node:crypto';
34
+import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
35
+import { join, relative, resolve } from 'node:path';
36
+import { parseArgs } from 'node:util';
37
+import { startJourneyContainer } from '@e2e-tests/utils';
38
+import { z } from 'zod';
39
+import { anthropicAdapter } from './src/anthropic-adapter';
40
+import {
41
+ AGENTS,
42
+ ARMS,
43
+ fingerprint,
44
+ previousRuns,
45
+ readBatch,
46
+ renderComparison,
47
+ renderSummary,
48
+ treeHash,
49
+ type AgentName,
50
+ type Arm,
51
+ type BatchInfo,
52
+ type RunRecord,
53
+} from './src/batch';
54
+import { CLAUDE_CODE_VERSION, claudeCodeDriver, ensureClaudeCodeBinary } from './src/claude-code';
55
+import { MAX_OUTPUT_TOKENS } from './src/adapters';
56
+import { JOURNEYS, type Journey, type JourneyRun } from './src/journeys';
57
+import { ALLOWED_MODELS, startModelProxy, type ModelProxy } from './src/model-proxy';
58
+import { loopDriver, runJourneyWith, type Driver } from './src/run-journey';
59
+import { buildSkill } from './src/skill';
60
+
61
+const HERE = import.meta.dir;
62
+const RESULTS = join(HERE, 'results');
63
+const REPO_TS = resolve(HERE, '..', '..', '..');
64
+/**
65
+ * What drives and grades a run: the journeys and their checks, the person,
66
+ * the agents, and the fake backend. A change here makes earlier results
67
+ * incomparable. The reporting code (`batch.ts`, `metrics.ts`) is left out.
68
+ */
69
+const HARNESS_PATHS = [
70
+ 'e2e-tests/cli/fake-backend/src',
71
+ ...[
72
+ 'agent.ts',
73
+ 'anthropic-adapter.ts',
74
+ 'claude-code.ts',
75
+ 'journeys.ts',
76
+ 'person.ts',
77
+ 'run-journey.ts',
78
+ 'scrub.ts',
79
+ 'skill.ts',
80
+ ].map(file => `e2e-tests/cli/agent-journeys/src/${file}`),
81
+];
82
+/** Wall-clock limit for one run's agent. */
83
+const RUN_DEADLINE_MS = 6 * 60_000;
84
+/** Shell commands one run may start. The longest recorded run used eleven. */
85
+const MAX_COMMANDS = 40;
86
+
87
+const fail = (message: string): never => {
88
+ console.error(message);
89
+ process.exit(2);
90
+};
91
+
92
+const git = (...args: string[]): string =>
93
+ execFileSync('git', args, { cwd: HERE, encoding: 'utf8' }).trim();
94
+
95
+/** A batch id under `results/`, or the path of a batch directory kept elsewhere (`evals/`). */
96
+const batchDirectory = (name: string): string =>
97
+ existsSync(join(name, 'batch.json')) ? resolve(name) : resolve(RESULTS, name);
98
+
99
+const [command, ...rest] = process.argv.slice(2);
100
+
101
+if (command === 'report') {
102
+ const [batch] = rest;
103
+ if (!batch) fail('Usage: bun run eval.ts report <batch>');
104
+ const { info, runs } = readBatch(batchDirectory(batch!));
105
+ process.stdout.write(renderSummary(info, runs));
106
+ process.exit(0);
107
+}
108
+
109
+if (command === 'keep') {
110
+ const [batch, decision] = rest;
111
+ if (!batch || !decision || !/^[a-z0-9][a-z0-9-]*$/.test(decision)) {
112
+ fail('Usage: bun run eval.ts keep <batch> <decision-name>');
113
+ }
114
+ const source = batchDirectory(batch!);
115
+ const { info } = readBatch(source);
116
+ const target = join(HERE, 'evals', decision!, info.id);
117
+ mkdirSync(target, { recursive: true });
118
+ // The records and the summary, not the logs: they are large and hold every command's output.
119
+ for (const name of ['batch.json', 'runs.jsonl', 'summary.md']) {
120
+ const text = readFileSync(join(source, name), 'utf8');
121
+ if (/sk-ant-|sk-journey-/.test(text)) fail(`${name} holds a key or a proxy token; not kept.`);
122
+ writeFileSync(join(target, name), text);
123
+ }
124
+ console.log(`kept ${relative(process.cwd(), target)}`);
125
+ process.exit(0);
126
+}
127
+
128
+if (command === 'compare') {
129
+ const { values, positionals } = parseArgs({
130
+ args: rest,
131
+ allowPositionals: true,
132
+ options: {
133
+ arm: { type: 'string', default: 'skill' },
134
+ 'allow-harness-change': { type: 'boolean', default: false },
135
+ },
136
+ });
137
+ const [baseline, candidate] = positionals;
138
+ const arm = z.enum(ARMS).safeParse(values.arm);
139
+ if (!baseline || !candidate || !arm.success) {
140
+ fail(
141
+ 'Usage: bun run eval.ts compare <baseline batch> <candidate batch> [--arm skill|no-skill]'
142
+ );
143
+ }
144
+ process.stdout.write(
145
+ renderComparison(readBatch(batchDirectory(baseline!)), readBatch(batchDirectory(candidate!)), {
146
+ arm: arm.data!,
147
+ allowHarnessChange: values['allow-harness-change'] ?? false,
148
+ })
149
+ );
150
+ process.exit(0);
151
+}
152
+
153
+if (command !== 'run') {
154
+ fail('Usage: bun run eval.ts run|report|compare|keep … (see the header of eval.ts)');
155
+}
156
+
157
+const { values } = parseArgs({
158
+ args: rest,
159
+ options: {
160
+ model: { type: 'string' },
161
+ agent: { type: 'string', default: 'claude-code' },
162
+ runs: { type: 'string', default: '3' },
163
+ arms: { type: 'string', default: 'skill' },
164
+ split: { type: 'string', default: 'all' },
165
+ journeys: { type: 'string' },
166
+ 'max-cost-usd': { type: 'string', default: '2' },
167
+ 'run-budget-usd': { type: 'string', default: '0.5' },
168
+ concurrency: { type: 'string', default: '2' },
169
+ label: { type: 'string' },
170
+ force: { type: 'boolean', default: false },
171
+ dry: { type: 'boolean', default: false },
172
+ smoke: { type: 'boolean', default: false },
173
+ },
174
+});
175
+
176
+const Options = z.object({
177
+ model: z.enum(ALLOWED_MODELS),
178
+ agent: z.enum(AGENTS),
179
+ runs: z.coerce.number().int().min(1).max(20),
180
+ arms: z.array(z.enum(ARMS)).min(1),
181
+ split: z.enum(['train', 'holdout', 'all']),
182
+ maxCostUsd: z.coerce.number().positive(),
183
+ runBudgetUsd: z.coerce.number().positive(),
184
+ concurrency: z.coerce.number().int().min(1).max(8),
185
+ label: z
186
+ .string()
187
+ .regex(/^[a-z0-9][a-z0-9-]*$/)
188
+ .optional(),
189
+});
190
+const parsed = Options.safeParse({
191
+ model: values.model,
192
+ agent: values.agent,
193
+ runs: values.smoke ? '1' : values.runs,
194
+ arms: values.arms?.split(',').map(arm => arm.trim()),
195
+ split: values.split,
196
+ maxCostUsd: values['max-cost-usd'],
197
+ runBudgetUsd: values['run-budget-usd'],
198
+ concurrency: values.concurrency,
199
+ label: values.label,
200
+});
201
+if (!parsed.success) {
202
+ fail(`${z.prettifyError(parsed.error)}\n\nModels: ${ALLOWED_MODELS.join(', ')}`);
203
+}
204
+const options = parsed.data!;
205
+
206
+const named = values.journeys?.split(',').map(id => id.trim());
207
+const unknown = named?.filter(id => !JOURNEYS.some(journey => journey.id === id)) ?? [];
208
+if (unknown.length > 0) fail(`Unknown journeys: ${unknown.join(', ')}`);
209
+let journeys: ReadonlyArray<Journey> = JOURNEYS.filter(
210
+ journey =>
211
+ (named === undefined || named.includes(journey.id)) &&
212
+ (options.split === 'all' || journey.split === options.split)
213
+);
214
+if (values.smoke) journeys = journeys.slice(0, 1);
215
+if (journeys.length === 0) fail('No journey matches.');
216
+
217
+const apiKey = process.env.ANTHROPIC_API_KEY;
218
+if (!apiKey && !values.dry) fail('ANTHROPIC_API_KEY is not set; nothing was started.');
219
+
220
+// What the batch measures. The CLI is the binary a journey container would
221
+// run now: starting one builds the image when the checkout changed. The
222
+// binary is hashed, not the image, which also holds the test sources.
223
+const probe = await startJourneyContainer({ label: 'eval-probe' });
224
+const binaries = await probe.exec('cd /usr/local/bin && sha256sum composio *.mjs | sha256sum', {
225
+ cwd: '/',
226
+});
227
+await probe.stop();
228
+if (binaries.exitCode !== 0) fail(`Could not hash the CLI binary: ${binaries.stderr}`);
229
+const cliSha256 = binaries.stdout.split(' ')[0]!.trim();
230
+
231
+const agentVersion = options.agent === 'claude-code' ? CLAUDE_CODE_VERSION : 'loop';
232
+const skillSha256 = createHash('sha256').update(JSON.stringify(buildSkill())).digest('hex');
233
+const harnessSha256 = treeHash(REPO_TS, HARNESS_PATHS);
234
+const startedAt = new Date();
235
+const batchId = `${startedAt.toISOString().replace(/[:.]/g, '-').slice(0, 19)}-${options.agent}-${options.model}${options.label ? `-${options.label}` : ''}`;
236
+const info: BatchInfo = {
237
+ id: batchId,
238
+ startedAt: startedAt.toISOString(),
239
+ agent: options.agent,
240
+ agentVersion,
241
+ model: options.model,
242
+ runs: options.runs,
243
+ arms: options.arms,
244
+ journeys: journeys.map(journey => journey.id),
245
+ revision: git('rev-parse', '--short', 'HEAD'),
246
+ dirty: git('status', '--porcelain', '--', ':/ts/packages/cli', ':/ts/e2e-tests/cli') !== '',
247
+ skillSha256,
248
+ harnessSha256,
249
+ cliSha256,
250
+ maxCostUsd: options.maxCostUsd,
251
+};
252
+
253
+interface Planned {
254
+ readonly journey: Journey;
255
+ readonly arm: Arm;
256
+ readonly attempt: number;
257
+ readonly fingerprint: string;
258
+}
259
+
260
+const earlier = values.force ? new Map<string, RunRecord[]>() : previousRuns(RESULTS);
261
+const reused: RunRecord[] = [];
262
+const planned: Planned[] = [];
263
+for (const journey of journeys) {
264
+ for (const arm of options.arms) {
265
+ const print = fingerprint({
266
+ journey: journey.id,
267
+ arm,
268
+ agent: options.agent,
269
+ agentVersion,
270
+ model: options.model,
271
+ skillSha256,
272
+ harnessSha256,
273
+ cliSha256,
274
+ });
275
+ const available = (earlier.get(print) ?? []).slice(-options.runs);
276
+ reused.push(...available.map((run, index) => ({ ...run, attempt: index + 1 })));
277
+ for (let attempt = available.length + 1; attempt <= options.runs; attempt += 1) {
278
+ planned.push({ journey, arm, attempt, fingerprint: print });
279
+ }
280
+ }
281
+}
282
+
283
+console.log(`batch ${batchId}`);
284
+console.log(`agent ${options.agent} ${agentVersion}, ${options.model}`);
285
+console.log(`journeys ${journeys.map(journey => `${journey.id} (${journey.split})`).join(', ')}`);
286
+console.log(`arms ${options.arms.join(', ')}; ${options.runs} run(s) each`);
287
+console.log(
288
+ `runs ${planned.length} new, ${reused.length} reused; budget $${options.maxCostUsd}`
289
+);
290
+if (values.dry) {
291
+ for (const run of planned)
292
+ console.log(` would run ${run.journey.id} ${run.arm} #${run.attempt}`);
293
+ process.exit(0);
294
+}
295
+
296
+const directory = join(RESULTS, batchId);
297
+mkdirSync(join(directory, 'logs'), { recursive: true });
298
+writeFileSync(join(directory, 'batch.json'), `${JSON.stringify(info, null, 2)}\n`);
299
+const record = (run: RunRecord) =>
300
+ appendFileSync(join(directory, 'runs.jsonl'), `${JSON.stringify(run)}\n`);
301
+for (const run of reused) record(run);
302
+
303
+const proxy: ModelProxy = await startModelProxy({ apiKey: apiKey! });
304
+const binary = options.agent === 'claude-code' ? ensureClaudeCodeBinary() : undefined;
305
+let spent = 0;
306
+let interrupted = false;
307
+process.on('SIGINT', () => {
308
+ interrupted = true;
309
+ console.log('\ninterrupted: finishing the runs in flight, starting no more');
310
+});
311
+
312
+/** Nothing that could carry a credential is written. */
313
+const redact = (text: string): string => {
314
+ const cleaned = text.replaceAll(proxy.token, '<proxy-token>');
315
+ if (cleaned.includes(apiKey!) || /sk-ant-/.test(cleaned)) {
316
+ throw new Error('Refusing to write a log that contains an API key.');
317
+ }
318
+ return cleaned;
319
+};
320
+
321
+const driverFor = (arm: Arm, budgetUsd: number, onEvent: (event: unknown) => void): Driver =>
322
+ options.agent === 'claude-code'
323
+ ? claudeCodeDriver({
324
+ model: options.model,
325
+ proxy,
326
+ binary: binary!,
327
+ withSkill: arm === 'skill',
328
+ budgetUsd,
329
+ deadlineMs: RUN_DEADLINE_MS,
330
+ maxCommands: MAX_COMMANDS,
331
+ onEvent,
332
+ })
333
+ : loopDriver(
334
+ inputs =>
335
+ anthropicAdapter({
336
+ baseURL: 'https://api.anthropic.com',
337
+ apiKey: apiKey!,
338
+ model: options.model,
339
+ maxTokens: MAX_OUTPUT_TOKENS,
340
+ ...inputs,
341
+ }),
342
+ { skill: arm === 'skill' ? 'system-prompt' : 'none', scrubbed: false }
343
+ );
344
+
345
+const runOne = async (plan: Planned): Promise<void> => {
346
+ const name = `${plan.journey.id}.${plan.arm}.${plan.attempt}`;
347
+ const remaining = options.maxCostUsd - spent;
348
+ if (interrupted || remaining <= 0) {
349
+ console.log(` skipped ${name}: ${interrupted ? 'interrupted' : 'budget spent'}`);
350
+ return;
351
+ }
352
+ const events: unknown[] = [];
353
+ const started = Date.now();
354
+ const refusedBefore = proxy.refused().length;
355
+ let run: JourneyRun | undefined;
356
+ try {
357
+ run = await runJourneyWith(
358
+ plan.journey,
359
+ driverFor(plan.arm, Math.min(options.runBudgetUsd, remaining), event => events.push(event))
360
+ );
361
+ const checks = plan.journey.checks(run);
362
+ const detours = (plan.journey.detours?.(run) ?? []).filter(detour => !detour.ok);
363
+ const stop =
364
+ typeof run.transcript.stop === 'string' ? run.transcript.stop : run.transcript.stop.provider;
365
+ // A harness or provider failure says nothing about the agent.
366
+ const refused = proxy.refused().slice(refusedBefore);
367
+ const infra = stop.startsWith('exited ') || refused.length > 0;
368
+ const costUsd = run.usage?.costUsd ?? 0;
369
+ spent += costUsd;
370
+ const result: RunRecord = {
371
+ journey: plan.journey.id,
372
+ split: plan.journey.split,
373
+ arm: plan.arm,
374
+ attempt: plan.attempt,
375
+ fingerprint: plan.fingerprint,
376
+ outcome: infra ? 'infra' : checks.every(check => check.ok) ? 'passed' : 'failed',
377
+ checks: checks.map(check => ({
378
+ name: check.name,
379
+ ok: check.ok,
380
+ ...(check.ok ? {} : { seen: JSON.stringify(check.seen ?? null).slice(0, 500) }),
381
+ })),
382
+ detours: detours.map(detour => ({
383
+ name: detour.name,
384
+ ok: false,
385
+ seen: JSON.stringify(detour.seen ?? null).slice(0, 500),
386
+ })),
387
+ stop,
388
+ modelTurns: run.transcript.modelTurns,
389
+ commands: run.transcript.commands.map(item => ({
390
+ command: item.command,
391
+ exitCode: item.exitCode,
392
+ })),
393
+ costUsd,
394
+ durationMs: Date.now() - started,
395
+ ...(infra ? { error: `stop: ${stop}; proxy refused: ${refused.join('; ')}` } : {}),
396
+ };
397
+ writeFileSync(
398
+ join(directory, 'logs', `${name}.json`),
399
+ redact(
400
+ JSON.stringify(
401
+ { ...result, transcript: run.transcript, requests: run.requests, events },
402
+ null,
403
+ 2
404
+ )
405
+ )
406
+ );
407
+ record(result);
408
+ console.log(
409
+ ` ${result.outcome.padEnd(6)} ${name} $${costUsd.toFixed(3)} ${result.commands.length} cmd ${detours.length} detour(s)`
410
+ );
411
+ } catch (error) {
412
+ const message = error instanceof Error ? error.message : String(error);
413
+ record({
414
+ journey: plan.journey.id,
415
+ split: plan.journey.split,
416
+ arm: plan.arm,
417
+ attempt: plan.attempt,
418
+ fingerprint: plan.fingerprint,
419
+ outcome: 'infra',
420
+ checks: [],
421
+ detours: [],
422
+ stop: 'error',
423
+ modelTurns: 0,
424
+ commands: [],
425
+ costUsd: 0,
426
+ durationMs: Date.now() - started,
427
+ error: redact(message).slice(0, 1000),
428
+ });
429
+ console.log(` infra ${name} ${message.slice(0, 200)}`);
430
+ } finally {
431
+ await run?.rig.stop();
432
+ }
433
+};
434
+
435
+try {
436
+ const queue = [...planned];
437
+ await Promise.all(
438
+ Array.from({ length: Math.min(options.concurrency, queue.length) }, async () => {
439
+ for (let next = queue.shift(); next !== undefined; next = queue.shift()) await runOne(next);
440
+ })
441
+ );
442
+} finally {
443
+ await proxy.close();
444
+}
445
+
446
+const summary = renderSummary(info, readBatch(directory).runs);
447
+writeFileSync(join(directory, 'summary.md'), summary);
448
+process.stdout.write(`\n${summary}`);
449
+console.log(`results ${directory}`);