Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Claude Code as the agent of a journey. |
| 3 | * |
| 4 | * The minimal loop in `agent.ts` puts the skill in its system prompt, so it |
| 5 | * measures whether the skill's text steers a model that has already read it. |
| 6 | * A user's agent finds the skill on its own, among others, from its |
| 7 | * description. This driver runs the real harness inside the journey container, |
| 8 | * headless, with the skill installed where Claude Code looks for it, so a run |
| 9 | * also measures whether the skill is found and used. |
| 10 | * |
| 11 | * The binary is the Linux build Anthropic publishes to npm, fetched once per |
| 12 | * version and mounted read-only. The container reaches the model only through |
| 13 | * `model-proxy.ts`: it holds a one-off token, never the API key. |
| 14 | */ |
| 15 | |
| 16 | import type { JournalEntry } from '@e2e-tests/cli-fake-backend'; |
| 17 | import { execFileSync } from 'node:child_process'; |
| 18 | import { chmodSync, existsSync, mkdirSync, mkdtempSync, renameSync, rmSync } from 'node:fs'; |
| 19 | import { homedir, tmpdir } from 'node:os'; |
| 20 | import { join } from 'node:path'; |
| 21 | import { z } from 'zod'; |
| 22 | import type { ExecutedCommand, StopReason } from './agent'; |
| 23 | import type { ModelProxy } from './model-proxy'; |
| 24 | import type { Driver } from './run-journey'; |
| 25 | import { addTokens, ZERO_USAGE, type Usage } from './usage'; |
| 26 | |
| 27 | /** Pinned so two runs of the same skill measure the same harness. */ |
| 28 | export const CLAUDE_CODE_VERSION = '2.1.310'; |
| 29 | |
| 30 | const CONTAINER_BINARY = '/opt/claude-code/claude'; |
| 31 | |
| 32 | /** How long Claude Code may take to exit once the run is over. */ |
| 33 | const END_GRACE_MS = 10_000; |
| 34 | |
| 35 | /** The tools the agent gets: a shell, reading files, and skills. No web, no subagents. */ |
| 36 | const TOOLS = 'Bash,Read,Glob,Grep,Skill'; |
| 37 | |
| 38 | /** The Linux architecture of the Docker engine, in npm's naming. */ |
| 39 | const dockerArchitecture = (): 'arm64' | 'x64' => { |
| 40 | const reported = execFileSync('docker', ['info', '--format', '{{.Architecture}}'], { |
| 41 | encoding: 'utf8', |
| 42 | }).trim(); |
| 43 | if (reported === 'aarch64' || reported === 'arm64') return 'arm64'; |
| 44 | if (reported === 'x86_64' || reported === 'amd64') return 'x64'; |
| 45 | throw new Error(`Unsupported Docker architecture: ${reported}`); |
| 46 | }; |
| 47 | |
| 48 | /** |
| 49 | * The host path of the Linux `claude` binary, downloaded on first use into |
| 50 | * `~/.cache/composio-e2e/claude-code/`. `npm pack` checks the package against |
| 51 | * the registry's integrity hash. |
| 52 | */ |
| 53 | export const ensureClaudeCodeBinary = (): string => { |
| 54 | const architecture = dockerArchitecture(); |
| 55 | const directory = join( |
| 56 | homedir(), |
| 57 | '.cache', |
| 58 | 'composio-e2e', |
| 59 | 'claude-code', |
| 60 | `${CLAUDE_CODE_VERSION}-linux-${architecture}` |
| 61 | ); |
| 62 | const binary = join(directory, 'claude'); |
| 63 | if (existsSync(binary)) return binary; |
| 64 | |
| 65 | const scratch = mkdtempSync(join(tmpdir(), 'composio-e2e-claude-code-')); |
| 66 | try { |
| 67 | const packageName = `@anthropic-ai/claude-code-linux-${architecture}@${CLAUDE_CODE_VERSION}`; |
| 68 | const tarball = execFileSync( |
| 69 | 'npm', |
| 70 | ['pack', packageName, '--pack-destination', scratch, '--silent'], |
| 71 | { encoding: 'utf8' } |
| 72 | ) |
| 73 | .trim() |
| 74 | .split('\n') |
| 75 | .at(-1); |
| 76 | if (!tarball) throw new Error(`npm pack ${packageName} printed no file name`); |
| 77 | execFileSync('tar', ['xzf', join(scratch, tarball), '-C', scratch]); |
| 78 | mkdirSync(directory, { recursive: true }); |
| 79 | renameSync(join(scratch, 'package', 'claude'), binary); |
| 80 | chmodSync(binary, 0o755); |
| 81 | return binary; |
| 82 | } finally { |
| 83 | rmSync(scratch, { recursive: true, force: true }); |
| 84 | } |
| 85 | }; |
| 86 | |
| 87 | const ContentBlock = z.looseObject({ |
| 88 | type: z.string(), |
| 89 | id: z.string().optional(), |
| 90 | name: z.string().optional(), |
| 91 | text: z.string().optional(), |
| 92 | input: z.unknown().optional(), |
| 93 | tool_use_id: z.string().optional(), |
| 94 | content: z.unknown().optional(), |
| 95 | is_error: z.boolean().optional(), |
| 96 | }); |
| 97 | |
| 98 | const StreamEvent = z.looseObject({ |
| 99 | type: z.string(), |
| 100 | subtype: z.string().optional(), |
| 101 | message: z.looseObject({ content: z.union([z.string(), z.array(ContentBlock)]) }).optional(), |
| 102 | tool_use_result: z.unknown().optional(), |
| 103 | result: z.string().optional(), |
| 104 | is_error: z.boolean().optional(), |
| 105 | num_turns: z.number().optional(), |
| 106 | total_cost_usd: z.number().optional(), |
| 107 | usage: z |
| 108 | .looseObject({ |
| 109 | input_tokens: z.number().optional(), |
| 110 | output_tokens: z.number().optional(), |
| 111 | cache_read_input_tokens: z.number().optional(), |
| 112 | cache_creation_input_tokens: z.number().optional(), |
| 113 | }) |
| 114 | .optional(), |
| 115 | }); |
| 116 | |
| 117 | const BashInput = z.looseObject({ command: z.string() }); |
| 118 | const BashResult = z.looseObject({ stdout: z.string(), stderr: z.string() }); |
| 119 | const TextPart = z.looseObject({ type: z.literal('text'), text: z.string() }); |
| 120 | |
| 121 | /** A tool result's text, whether Claude Code sent a string or text blocks. */ |
| 122 | const resultText = (content: unknown): string => { |
| 123 | if (typeof content === 'string') return content; |
| 124 | if (!Array.isArray(content)) return ''; |
| 125 | return content |
| 126 | .flatMap(part => { |
| 127 | const parsed = TextPart.safeParse(part); |
| 128 | return parsed.success ? [parsed.data.text] : []; |
| 129 | }) |
| 130 | .join('\n'); |
| 131 | }; |
| 132 | |
| 133 | const userMessage = (text: string): string => |
| 134 | `${JSON.stringify({ type: 'user', message: { role: 'user', content: text } })}\n`; |
| 135 | |
| 136 | export interface ClaudeCodeOptions { |
| 137 | readonly model: string; |
| 138 | readonly proxy: ModelProxy; |
| 139 | readonly binary: string; |
| 140 | /** Install the skill (`true`), or run without it. */ |
| 141 | readonly withSkill: boolean; |
| 142 | /** Claude Code stops itself once a run has cost this much. */ |
| 143 | readonly budgetUsd: number; |
| 144 | /** Wall-clock limit for the agent; the run is cut off after it. */ |
| 145 | readonly deadlineMs: number; |
| 146 | /** Cut-off for shell commands, so a looping agent cannot run forever. */ |
| 147 | readonly maxCommands: number; |
| 148 | /** Every stream event, for the run's log. */ |
| 149 | readonly onEvent?: (event: unknown) => void; |
| 150 | } |
| 151 | |
| 152 | export const claudeCodeDriver = (options: ClaudeCodeOptions): Driver => ({ |
| 153 | skill: options.withSkill ? 'claude-code-skills' : 'none', |
| 154 | scrubbed: false, |
| 155 | readOnlyMounts: [{ source: options.binary, target: CONTAINER_BINARY }], |
| 156 | drive: async ({ rig, journey, person }) => { |
| 157 | const environment: Record<string, string> = { |
| 158 | ANTHROPIC_BASE_URL: options.proxy.dockerUrl, |
| 159 | ANTHROPIC_API_KEY: options.proxy.token, |
| 160 | // Lets permissions be skipped as root; the container is the sandbox. |
| 161 | IS_SANDBOX: '1', |
| 162 | CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', |
| 163 | DISABLE_AUTOUPDATER: '1', |
| 164 | // Every model Claude Code might pick for a side task is the run's model, |
| 165 | // or Haiku, so the proxy's allow-list holds. |
| 166 | ANTHROPIC_SMALL_FAST_MODEL: 'claude-haiku-5-5', |
| 167 | ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-haiku-5-5', |
| 168 | ANTHROPIC_DEFAULT_SONNET_MODEL: options.model, |
| 169 | ANTHROPIC_DEFAULT_OPUS_MODEL: options.model, |
| 170 | CLAUDE_CODE_SUBAGENT_MODEL: options.model, |
| 171 | }; |
| 172 | const child = Bun.spawn( |
| 173 | [ |
| 174 | 'docker', |
| 175 | 'exec', |
| 176 | '-i', |
| 177 | '--workdir', |
| 178 | journey.cwd ?? rig.journey.home, |
| 179 | ...Object.entries(environment).flatMap(([name, value]) => ['-e', `${name}=${value}`]), |
| 180 | rig.journey.name, |
| 181 | CONTAINER_BINARY, |
| 182 | '-p', |
| 183 | '--input-format', |
| 184 | 'stream-json', |
| 185 | '--output-format', |
| 186 | 'stream-json', |
| 187 | '--verbose', |
| 188 | '--model', |
| 189 | options.model, |
| 190 | '--permission-mode', |
| 191 | 'bypassPermissions', |
| 192 | '--tools', |
| 193 | TOOLS, |
| 194 | '--no-session-persistence', |
| 195 | '--max-budget-usd', |
| 196 | String(options.budgetUsd), |
| 197 | ], |
| 198 | { stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' } |
| 199 | ); |
| 200 | const send = (text: string) => { |
| 201 | child.stdin.write(userMessage(text)); |
| 202 | void child.stdin.flush(); |
| 203 | }; |
| 204 | |
| 205 | const commands: ExecutedCommand[] = []; |
| 206 | const commandRequests: Array<ReadonlyArray<JournalEntry>> = []; |
| 207 | const messages: string[] = []; |
| 208 | const replies: Array<{ afterModelTurn: number; text: string }> = []; |
| 209 | /** Bash calls that have started, by tool-use id, with the journal position at their start. */ |
| 210 | const running = new Map<string, { command: string; mark: number }>(); |
| 211 | let usage: Usage = ZERO_USAGE; |
| 212 | let modelTurns = 0; |
| 213 | let stop: StopReason | undefined; |
| 214 | let finished = false; |
| 215 | |
| 216 | const cutOff = (reason: string) => { |
| 217 | if (stop === undefined) stop = { provider: reason }; |
| 218 | child.kill(); |
| 219 | }; |
| 220 | const deadline = setTimeout(() => cutOff('deadline'), options.deadlineMs); |
| 221 | let ending: ReturnType<typeof setTimeout> | undefined; |
| 222 | /** |
| 223 | * Closing stdin ends the session, but a command the agent left running in |
| 224 | * the background keeps Claude Code alive. The run is over once the agent |
| 225 | * stopped and the person had nothing to say, so it gets a moment, then goes. |
| 226 | */ |
| 227 | const end = () => { |
| 228 | child.stdin.end(); |
| 229 | ending ??= setTimeout(() => child.kill(), END_GRACE_MS); |
| 230 | }; |
| 231 | |
| 232 | const handle = (event: z.infer<typeof StreamEvent>) => { |
| 233 | const content = event.message?.content; |
| 234 | const blocks = Array.isArray(content) ? content : []; |
| 235 | if (event.type === 'assistant') { |
| 236 | const text = blocks |
| 237 | .filter(block => block.type === 'text' && block.text !== undefined) |
| 238 | .map(block => block.text) |
| 239 | .join('\n'); |
| 240 | if (text !== '') { |
| 241 | messages.push(text); |
| 242 | // The person reads what the agent writes as it writes it. |
| 243 | person.hear(text, false); |
| 244 | } |
| 245 | for (const block of blocks) { |
| 246 | if (block.type !== 'tool_use' || block.name !== 'Bash' || block.id === undefined) |
| 247 | continue; |
| 248 | const input = BashInput.safeParse(block.input); |
| 249 | if (!input.success) continue; |
| 250 | running.set(block.id, { command: input.data.command, mark: rig.fake.journal.mark() }); |
| 251 | if (running.size + commands.length > options.maxCommands) cutOff('command-limit'); |
| 252 | } |
| 253 | } |
| 254 | if (event.type === 'user') { |
| 255 | for (const block of blocks) { |
| 256 | if (block.type !== 'tool_result' || block.tool_use_id === undefined) continue; |
| 257 | const started = running.get(block.tool_use_id); |
| 258 | if (!started) continue; |
| 259 | running.delete(block.tool_use_id); |
| 260 | const payload = resultText(block.content); |
| 261 | const exitCode = block.is_error ? Number(/^Exit code (\d+)/.exec(payload)?.[1] ?? 1) : 0; |
| 262 | const output = BashResult.safeParse(event.tool_use_result); |
| 263 | commands.push({ |
| 264 | command: started.command, |
| 265 | exitCode, |
| 266 | stdout: output.success ? output.data.stdout : payload.replace(/^Exit code \d+\n?/, ''), |
| 267 | stderr: output.success ? output.data.stderr : '', |
| 268 | payload, |
| 269 | }); |
| 270 | // Requests between the call and its result. Calls that overlap share requests. |
| 271 | commandRequests.push(rig.fake.journal.since(started.mark)); |
| 272 | } |
| 273 | } |
| 274 | if (event.type === 'result') { |
| 275 | modelTurns += event.num_turns ?? 0; |
| 276 | usage = { |
| 277 | ...addTokens(usage, '', { |
| 278 | input: event.usage?.input_tokens ?? 0, |
| 279 | output: event.usage?.output_tokens ?? 0, |
| 280 | cacheRead: event.usage?.cache_read_input_tokens ?? 0, |
| 281 | cacheWrite: event.usage?.cache_creation_input_tokens ?? 0, |
| 282 | }), |
| 283 | // Claude Code reports the session's running total. |
| 284 | costUsd: event.total_cost_usd ?? usage.costUsd, |
| 285 | }; |
| 286 | if (event.subtype !== 'success') { |
| 287 | stop = { provider: event.subtype ?? 'error' }; |
| 288 | end(); |
| 289 | return; |
| 290 | } |
| 291 | const reply = person.hear(event.result ?? '', true); |
| 292 | if (reply === undefined) { |
| 293 | finished = true; |
| 294 | end(); |
| 295 | return; |
| 296 | } |
| 297 | replies.push({ afterModelTurn: modelTurns, text: reply }); |
| 298 | send(reply); |
| 299 | } |
| 300 | }; |
| 301 | |
| 302 | send(journey.prompt); |
| 303 | const decoder = new TextDecoder(); |
| 304 | let buffer = ''; |
| 305 | for await (const chunk of child.stdout) { |
| 306 | buffer += decoder.decode(chunk, { stream: true }); |
| 307 | for (let newline = buffer.indexOf('\n'); newline >= 0; newline = buffer.indexOf('\n')) { |
| 308 | const line = buffer.slice(0, newline).trim(); |
| 309 | buffer = buffer.slice(newline + 1); |
| 310 | if (line === '') continue; |
| 311 | const raw: unknown = JSON.parse(line); |
| 312 | options.onEvent?.(raw); |
| 313 | const event = StreamEvent.safeParse(raw); |
| 314 | if (event.success) handle(event.data); |
| 315 | } |
| 316 | } |
| 317 | const stderr = await new Response(child.stderr).text(); |
| 318 | await child.exited; |
| 319 | clearTimeout(deadline); |
| 320 | clearTimeout(ending); |
| 321 | if (stop === undefined && !finished) { |
| 322 | stop = { provider: `exited ${child.exitCode}: ${stderr.trim().slice(-300)}` }; |
| 323 | } |
| 324 | |
| 325 | return { |
| 326 | transcript: { |
| 327 | commands, |
| 328 | messages, |
| 329 | replies, |
| 330 | stop: stop ?? 'finished', |
| 331 | modelTurns, |
| 332 | }, |
| 333 | commandRequests, |
| 334 | usage, |
| 335 | }; |
| 336 | }, |
| 337 | }); |