Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Live, repeated journey runs: the measurement a skill change is judged by. |
| 3 | * |
| 4 | * bun run eval.ts run --model claude-haiku-5-5 [options] |
| 5 | * bun run eval.ts report <batch> |
| 6 | * bun run eval.ts compare <baseline batch> <candidate batch> [--arm skill] |
| 7 | * bun run eval.ts keep <batch> <decision-name> |
| 8 | * |
| 9 | * A batch is named by its id under `results/`, or by the path of a batch |
| 10 | * directory kept in the repository (see `evals/`). |
| 11 | * |
| 12 | * `run` options: |
| 13 | * --agent claude-code|loop Claude Code with the skill installed (default), or |
| 14 | * the minimal loop with the skill as system prompt |
| 15 | * --runs <k> runs per journey and arm (default 3) |
| 16 | * --arms skill,no-skill which arms to run (default skill) |
| 17 | * --split train|holdout|all which journeys (default all) |
| 18 | * --journeys <id,id> exactly these journeys |
| 19 | * --max-cost-usd <usd> stop starting runs once this much is spent (default 2) |
| 20 | * --run-budget-usd <usd> Claude Code stops a single run at this cost (default 0.5) |
| 21 | * --concurrency <n> runs at once (default 2) |
| 22 | * --label <name> appended to the batch id |
| 23 | * --force run everything, reusing no earlier run |
| 24 | * --dry print the plan; start nothing, spend nothing |
| 25 | * --smoke one journey, one run, to check the setup |
| 26 | * |
| 27 | * The API key is read from ANTHROPIC_API_KEY and stays in this process: the |
| 28 | * agent reaches the model through `src/model-proxy.ts`. Results go to |
| 29 | * `results/<batch>/` (git-ignored). See HILLCLIMB.md for how to use them. |
| 30 | */ |
| 31 | |
| 32 | import { execFileSync } from 'node:child_process'; |
| 33 | import { createHash } from 'node:crypto'; |
| 34 | import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; |
| 35 | import { join, relative, resolve } from 'node:path'; |
| 36 | import { parseArgs } from 'node:util'; |
| 37 | import { startJourneyContainer } from '@e2e-tests/utils'; |
| 38 | import { z } from 'zod'; |
| 39 | import { anthropicAdapter } from './src/anthropic-adapter'; |
| 40 | import { |
| 41 | AGENTS, |
| 42 | ARMS, |
| 43 | fingerprint, |
| 44 | previousRuns, |
| 45 | readBatch, |
| 46 | renderComparison, |
| 47 | renderSummary, |
| 48 | treeHash, |
| 49 | type AgentName, |
| 50 | type Arm, |
| 51 | type BatchInfo, |
| 52 | type RunRecord, |
| 53 | } from './src/batch'; |
| 54 | import { CLAUDE_CODE_VERSION, claudeCodeDriver, ensureClaudeCodeBinary } from './src/claude-code'; |
| 55 | import { MAX_OUTPUT_TOKENS } from './src/adapters'; |
| 56 | import { JOURNEYS, type Journey, type JourneyRun } from './src/journeys'; |
| 57 | import { ALLOWED_MODELS, startModelProxy, type ModelProxy } from './src/model-proxy'; |
| 58 | import { loopDriver, runJourneyWith, type Driver } from './src/run-journey'; |
| 59 | import { buildSkill } from './src/skill'; |
| 60 | |
| 61 | const HERE = import.meta.dir; |
| 62 | const RESULTS = join(HERE, 'results'); |
| 63 | const REPO_TS = resolve(HERE, '..', '..', '..'); |
| 64 | /** |
| 65 | * What drives and grades a run: the journeys and their checks, the person, |
| 66 | * the agents, and the fake backend. A change here makes earlier results |
| 67 | * incomparable. The reporting code (`batch.ts`, `metrics.ts`) is left out. |
| 68 | */ |
| 69 | const HARNESS_PATHS = [ |
| 70 | 'e2e-tests/cli/fake-backend/src', |
| 71 | ...[ |
| 72 | 'agent.ts', |
| 73 | 'anthropic-adapter.ts', |
| 74 | 'claude-code.ts', |
| 75 | 'journeys.ts', |
| 76 | 'person.ts', |
| 77 | 'run-journey.ts', |
| 78 | 'scrub.ts', |
| 79 | 'skill.ts', |
| 80 | ].map(file => `e2e-tests/cli/agent-journeys/src/${file}`), |
| 81 | ]; |
| 82 | /** Wall-clock limit for one run's agent. */ |
| 83 | const RUN_DEADLINE_MS = 6 * 60_000; |
| 84 | /** Shell commands one run may start. The longest recorded run used eleven. */ |
| 85 | const MAX_COMMANDS = 40; |
| 86 | |
| 87 | const fail = (message: string): never => { |
| 88 | console.error(message); |
| 89 | process.exit(2); |
| 90 | }; |
| 91 | |
| 92 | const git = (...args: string[]): string => |
| 93 | execFileSync('git', args, { cwd: HERE, encoding: 'utf8' }).trim(); |
| 94 | |
| 95 | /** A batch id under `results/`, or the path of a batch directory kept elsewhere (`evals/`). */ |
| 96 | const batchDirectory = (name: string): string => |
| 97 | existsSync(join(name, 'batch.json')) ? resolve(name) : resolve(RESULTS, name); |
| 98 | |
| 99 | const [command, ...rest] = process.argv.slice(2); |
| 100 | |
| 101 | if (command === 'report') { |
| 102 | const [batch] = rest; |
| 103 | if (!batch) fail('Usage: bun run eval.ts report <batch>'); |
| 104 | const { info, runs } = readBatch(batchDirectory(batch!)); |
| 105 | process.stdout.write(renderSummary(info, runs)); |
| 106 | process.exit(0); |
| 107 | } |
| 108 | |
| 109 | if (command === 'keep') { |
| 110 | const [batch, decision] = rest; |
| 111 | if (!batch || !decision || !/^[a-z0-9][a-z0-9-]*$/.test(decision)) { |
| 112 | fail('Usage: bun run eval.ts keep <batch> <decision-name>'); |
| 113 | } |
| 114 | const source = batchDirectory(batch!); |
| 115 | const { info } = readBatch(source); |
| 116 | const target = join(HERE, 'evals', decision!, info.id); |
| 117 | mkdirSync(target, { recursive: true }); |
| 118 | // The records and the summary, not the logs: they are large and hold every command's output. |
| 119 | for (const name of ['batch.json', 'runs.jsonl', 'summary.md']) { |
| 120 | const text = readFileSync(join(source, name), 'utf8'); |
| 121 | if (/sk-ant-|sk-journey-/.test(text)) fail(`${name} holds a key or a proxy token; not kept.`); |
| 122 | writeFileSync(join(target, name), text); |
| 123 | } |
| 124 | console.log(`kept ${relative(process.cwd(), target)}`); |
| 125 | process.exit(0); |
| 126 | } |
| 127 | |
| 128 | if (command === 'compare') { |
| 129 | const { values, positionals } = parseArgs({ |
| 130 | args: rest, |
| 131 | allowPositionals: true, |
| 132 | options: { |
| 133 | arm: { type: 'string', default: 'skill' }, |
| 134 | 'allow-harness-change': { type: 'boolean', default: false }, |
| 135 | }, |
| 136 | }); |
| 137 | const [baseline, candidate] = positionals; |
| 138 | const arm = z.enum(ARMS).safeParse(values.arm); |
| 139 | if (!baseline || !candidate || !arm.success) { |
| 140 | fail( |
| 141 | 'Usage: bun run eval.ts compare <baseline batch> <candidate batch> [--arm skill|no-skill]' |
| 142 | ); |
| 143 | } |
| 144 | process.stdout.write( |
| 145 | renderComparison(readBatch(batchDirectory(baseline!)), readBatch(batchDirectory(candidate!)), { |
| 146 | arm: arm.data!, |
| 147 | allowHarnessChange: values['allow-harness-change'] ?? false, |
| 148 | }) |
| 149 | ); |
| 150 | process.exit(0); |
| 151 | } |
| 152 | |
| 153 | if (command !== 'run') { |
| 154 | fail('Usage: bun run eval.ts run|report|compare|keep … (see the header of eval.ts)'); |
| 155 | } |
| 156 | |
| 157 | const { values } = parseArgs({ |
| 158 | args: rest, |
| 159 | options: { |
| 160 | model: { type: 'string' }, |
| 161 | agent: { type: 'string', default: 'claude-code' }, |
| 162 | runs: { type: 'string', default: '3' }, |
| 163 | arms: { type: 'string', default: 'skill' }, |
| 164 | split: { type: 'string', default: 'all' }, |
| 165 | journeys: { type: 'string' }, |
| 166 | 'max-cost-usd': { type: 'string', default: '2' }, |
| 167 | 'run-budget-usd': { type: 'string', default: '0.5' }, |
| 168 | concurrency: { type: 'string', default: '2' }, |
| 169 | label: { type: 'string' }, |
| 170 | force: { type: 'boolean', default: false }, |
| 171 | dry: { type: 'boolean', default: false }, |
| 172 | smoke: { type: 'boolean', default: false }, |
| 173 | }, |
| 174 | }); |
| 175 | |
| 176 | const Options = z.object({ |
| 177 | model: z.enum(ALLOWED_MODELS), |
| 178 | agent: z.enum(AGENTS), |
| 179 | runs: z.coerce.number().int().min(1).max(20), |
| 180 | arms: z.array(z.enum(ARMS)).min(1), |
| 181 | split: z.enum(['train', 'holdout', 'all']), |
| 182 | maxCostUsd: z.coerce.number().positive(), |
| 183 | runBudgetUsd: z.coerce.number().positive(), |
| 184 | concurrency: z.coerce.number().int().min(1).max(8), |
| 185 | label: z |
| 186 | .string() |
| 187 | .regex(/^[a-z0-9][a-z0-9-]*$/) |
| 188 | .optional(), |
| 189 | }); |
| 190 | const parsed = Options.safeParse({ |
| 191 | model: values.model, |
| 192 | agent: values.agent, |
| 193 | runs: values.smoke ? '1' : values.runs, |
| 194 | arms: values.arms?.split(',').map(arm => arm.trim()), |
| 195 | split: values.split, |
| 196 | maxCostUsd: values['max-cost-usd'], |
| 197 | runBudgetUsd: values['run-budget-usd'], |
| 198 | concurrency: values.concurrency, |
| 199 | label: values.label, |
| 200 | }); |
| 201 | if (!parsed.success) { |
| 202 | fail(`${z.prettifyError(parsed.error)}\n\nModels: ${ALLOWED_MODELS.join(', ')}`); |
| 203 | } |
| 204 | const options = parsed.data!; |
| 205 | |
| 206 | const named = values.journeys?.split(',').map(id => id.trim()); |
| 207 | const unknown = named?.filter(id => !JOURNEYS.some(journey => journey.id === id)) ?? []; |
| 208 | if (unknown.length > 0) fail(`Unknown journeys: ${unknown.join(', ')}`); |
| 209 | let journeys: ReadonlyArray<Journey> = JOURNEYS.filter( |
| 210 | journey => |
| 211 | (named === undefined || named.includes(journey.id)) && |
| 212 | (options.split === 'all' || journey.split === options.split) |
| 213 | ); |
| 214 | if (values.smoke) journeys = journeys.slice(0, 1); |
| 215 | if (journeys.length === 0) fail('No journey matches.'); |
| 216 | |
| 217 | const apiKey = process.env.ANTHROPIC_API_KEY; |
| 218 | if (!apiKey && !values.dry) fail('ANTHROPIC_API_KEY is not set; nothing was started.'); |
| 219 | |
| 220 | // What the batch measures. The CLI is the binary a journey container would |
| 221 | // run now: starting one builds the image when the checkout changed. The |
| 222 | // binary is hashed, not the image, which also holds the test sources. |
| 223 | const probe = await startJourneyContainer({ label: 'eval-probe' }); |
| 224 | const binaries = await probe.exec('cd /usr/local/bin && sha256sum composio *.mjs | sha256sum', { |
| 225 | cwd: '/', |
| 226 | }); |
| 227 | await probe.stop(); |
| 228 | if (binaries.exitCode !== 0) fail(`Could not hash the CLI binary: ${binaries.stderr}`); |
| 229 | const cliSha256 = binaries.stdout.split(' ')[0]!.trim(); |
| 230 | |
| 231 | const agentVersion = options.agent === 'claude-code' ? CLAUDE_CODE_VERSION : 'loop'; |
| 232 | const skillSha256 = createHash('sha256').update(JSON.stringify(buildSkill())).digest('hex'); |
| 233 | const harnessSha256 = treeHash(REPO_TS, HARNESS_PATHS); |
| 234 | const startedAt = new Date(); |
| 235 | const batchId = `${startedAt.toISOString().replace(/[:.]/g, '-').slice(0, 19)}-${options.agent}-${options.model}${options.label ? `-${options.label}` : ''}`; |
| 236 | const info: BatchInfo = { |
| 237 | id: batchId, |
| 238 | startedAt: startedAt.toISOString(), |
| 239 | agent: options.agent, |
| 240 | agentVersion, |
| 241 | model: options.model, |
| 242 | runs: options.runs, |
| 243 | arms: options.arms, |
| 244 | journeys: journeys.map(journey => journey.id), |
| 245 | revision: git('rev-parse', '--short', 'HEAD'), |
| 246 | dirty: git('status', '--porcelain', '--', ':/ts/packages/cli', ':/ts/e2e-tests/cli') !== '', |
| 247 | skillSha256, |
| 248 | harnessSha256, |
| 249 | cliSha256, |
| 250 | maxCostUsd: options.maxCostUsd, |
| 251 | }; |
| 252 | |
| 253 | interface Planned { |
| 254 | readonly journey: Journey; |
| 255 | readonly arm: Arm; |
| 256 | readonly attempt: number; |
| 257 | readonly fingerprint: string; |
| 258 | } |
| 259 | |
| 260 | const earlier = values.force ? new Map<string, RunRecord[]>() : previousRuns(RESULTS); |
| 261 | const reused: RunRecord[] = []; |
| 262 | const planned: Planned[] = []; |
| 263 | for (const journey of journeys) { |
| 264 | for (const arm of options.arms) { |
| 265 | const print = fingerprint({ |
| 266 | journey: journey.id, |
| 267 | arm, |
| 268 | agent: options.agent, |
| 269 | agentVersion, |
| 270 | model: options.model, |
| 271 | skillSha256, |
| 272 | harnessSha256, |
| 273 | cliSha256, |
| 274 | }); |
| 275 | const available = (earlier.get(print) ?? []).slice(-options.runs); |
| 276 | reused.push(...available.map((run, index) => ({ ...run, attempt: index + 1 }))); |
| 277 | for (let attempt = available.length + 1; attempt <= options.runs; attempt += 1) { |
| 278 | planned.push({ journey, arm, attempt, fingerprint: print }); |
| 279 | } |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | console.log(`batch ${batchId}`); |
| 284 | console.log(`agent ${options.agent} ${agentVersion}, ${options.model}`); |
| 285 | console.log(`journeys ${journeys.map(journey => `${journey.id} (${journey.split})`).join(', ')}`); |
| 286 | console.log(`arms ${options.arms.join(', ')}; ${options.runs} run(s) each`); |
| 287 | console.log( |
| 288 | `runs ${planned.length} new, ${reused.length} reused; budget $${options.maxCostUsd}` |
| 289 | ); |
| 290 | if (values.dry) { |
| 291 | for (const run of planned) |
| 292 | console.log(` would run ${run.journey.id} ${run.arm} #${run.attempt}`); |
| 293 | process.exit(0); |
| 294 | } |
| 295 | |
| 296 | const directory = join(RESULTS, batchId); |
| 297 | mkdirSync(join(directory, 'logs'), { recursive: true }); |
| 298 | writeFileSync(join(directory, 'batch.json'), `${JSON.stringify(info, null, 2)}\n`); |
| 299 | const record = (run: RunRecord) => |
| 300 | appendFileSync(join(directory, 'runs.jsonl'), `${JSON.stringify(run)}\n`); |
| 301 | for (const run of reused) record(run); |
| 302 | |
| 303 | const proxy: ModelProxy = await startModelProxy({ apiKey: apiKey! }); |
| 304 | const binary = options.agent === 'claude-code' ? ensureClaudeCodeBinary() : undefined; |
| 305 | let spent = 0; |
| 306 | let interrupted = false; |
| 307 | process.on('SIGINT', () => { |
| 308 | interrupted = true; |
| 309 | console.log('\ninterrupted: finishing the runs in flight, starting no more'); |
| 310 | }); |
| 311 | |
| 312 | /** Nothing that could carry a credential is written. */ |
| 313 | const redact = (text: string): string => { |
| 314 | const cleaned = text.replaceAll(proxy.token, '<proxy-token>'); |
| 315 | if (cleaned.includes(apiKey!) || /sk-ant-/.test(cleaned)) { |
| 316 | throw new Error('Refusing to write a log that contains an API key.'); |
| 317 | } |
| 318 | return cleaned; |
| 319 | }; |
| 320 | |
| 321 | const driverFor = (arm: Arm, budgetUsd: number, onEvent: (event: unknown) => void): Driver => |
| 322 | options.agent === 'claude-code' |
| 323 | ? claudeCodeDriver({ |
| 324 | model: options.model, |
| 325 | proxy, |
| 326 | binary: binary!, |
| 327 | withSkill: arm === 'skill', |
| 328 | budgetUsd, |
| 329 | deadlineMs: RUN_DEADLINE_MS, |
| 330 | maxCommands: MAX_COMMANDS, |
| 331 | onEvent, |
| 332 | }) |
| 333 | : loopDriver( |
| 334 | inputs => |
| 335 | anthropicAdapter({ |
| 336 | baseURL: 'https://api.anthropic.com', |
| 337 | apiKey: apiKey!, |
| 338 | model: options.model, |
| 339 | maxTokens: MAX_OUTPUT_TOKENS, |
| 340 | ...inputs, |
| 341 | }), |
| 342 | { skill: arm === 'skill' ? 'system-prompt' : 'none', scrubbed: false } |
| 343 | ); |
| 344 | |
| 345 | const runOne = async (plan: Planned): Promise<void> => { |
| 346 | const name = `${plan.journey.id}.${plan.arm}.${plan.attempt}`; |
| 347 | const remaining = options.maxCostUsd - spent; |
| 348 | if (interrupted || remaining <= 0) { |
| 349 | console.log(` skipped ${name}: ${interrupted ? 'interrupted' : 'budget spent'}`); |
| 350 | return; |
| 351 | } |
| 352 | const events: unknown[] = []; |
| 353 | const started = Date.now(); |
| 354 | const refusedBefore = proxy.refused().length; |
| 355 | let run: JourneyRun | undefined; |
| 356 | try { |
| 357 | run = await runJourneyWith( |
| 358 | plan.journey, |
| 359 | driverFor(plan.arm, Math.min(options.runBudgetUsd, remaining), event => events.push(event)) |
| 360 | ); |
| 361 | const checks = plan.journey.checks(run); |
| 362 | const detours = (plan.journey.detours?.(run) ?? []).filter(detour => !detour.ok); |
| 363 | const stop = |
| 364 | typeof run.transcript.stop === 'string' ? run.transcript.stop : run.transcript.stop.provider; |
| 365 | // A harness or provider failure says nothing about the agent. |
| 366 | const refused = proxy.refused().slice(refusedBefore); |
| 367 | const infra = stop.startsWith('exited ') || refused.length > 0; |
| 368 | const costUsd = run.usage?.costUsd ?? 0; |
| 369 | spent += costUsd; |
| 370 | const result: RunRecord = { |
| 371 | journey: plan.journey.id, |
| 372 | split: plan.journey.split, |
| 373 | arm: plan.arm, |
| 374 | attempt: plan.attempt, |
| 375 | fingerprint: plan.fingerprint, |
| 376 | outcome: infra ? 'infra' : checks.every(check => check.ok) ? 'passed' : 'failed', |
| 377 | checks: checks.map(check => ({ |
| 378 | name: check.name, |
| 379 | ok: check.ok, |
| 380 | ...(check.ok ? {} : { seen: JSON.stringify(check.seen ?? null).slice(0, 500) }), |
| 381 | })), |
| 382 | detours: detours.map(detour => ({ |
| 383 | name: detour.name, |
| 384 | ok: false, |
| 385 | seen: JSON.stringify(detour.seen ?? null).slice(0, 500), |
| 386 | })), |
| 387 | stop, |
| 388 | modelTurns: run.transcript.modelTurns, |
| 389 | commands: run.transcript.commands.map(item => ({ |
| 390 | command: item.command, |
| 391 | exitCode: item.exitCode, |
| 392 | })), |
| 393 | costUsd, |
| 394 | durationMs: Date.now() - started, |
| 395 | ...(infra ? { error: `stop: ${stop}; proxy refused: ${refused.join('; ')}` } : {}), |
| 396 | }; |
| 397 | writeFileSync( |
| 398 | join(directory, 'logs', `${name}.json`), |
| 399 | redact( |
| 400 | JSON.stringify( |
| 401 | { ...result, transcript: run.transcript, requests: run.requests, events }, |
| 402 | null, |
| 403 | 2 |
| 404 | ) |
| 405 | ) |
| 406 | ); |
| 407 | record(result); |
| 408 | console.log( |
| 409 | ` ${result.outcome.padEnd(6)} ${name} $${costUsd.toFixed(3)} ${result.commands.length} cmd ${detours.length} detour(s)` |
| 410 | ); |
| 411 | } catch (error) { |
| 412 | const message = error instanceof Error ? error.message : String(error); |
| 413 | record({ |
| 414 | journey: plan.journey.id, |
| 415 | split: plan.journey.split, |
| 416 | arm: plan.arm, |
| 417 | attempt: plan.attempt, |
| 418 | fingerprint: plan.fingerprint, |
| 419 | outcome: 'infra', |
| 420 | checks: [], |
| 421 | detours: [], |
| 422 | stop: 'error', |
| 423 | modelTurns: 0, |
| 424 | commands: [], |
| 425 | costUsd: 0, |
| 426 | durationMs: Date.now() - started, |
| 427 | error: redact(message).slice(0, 1000), |
| 428 | }); |
| 429 | console.log(` infra ${name} ${message.slice(0, 200)}`); |
| 430 | } finally { |
| 431 | await run?.rig.stop(); |
| 432 | } |
| 433 | }; |
| 434 | |
| 435 | try { |
| 436 | const queue = [...planned]; |
| 437 | await Promise.all( |
| 438 | Array.from({ length: Math.min(options.concurrency, queue.length) }, async () => { |
| 439 | for (let next = queue.shift(); next !== undefined; next = queue.shift()) await runOne(next); |
| 440 | }) |
| 441 | ); |
| 442 | } finally { |
| 443 | await proxy.close(); |
| 444 | } |
| 445 | |
| 446 | const summary = renderSummary(info, readBatch(directory).runs); |
| 447 | writeFileSync(join(directory, 'summary.md'), summary); |
| 448 | process.stdout.write(`\n${summary}`); |
| 449 | console.log(`results ${directory}`); |