Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Records one journey with a real model. |
| 3 | * |
| 4 | * ANTHROPIC_API_KEY=… bun run record.ts --set <name> --journey <id> --provider anthropic --model <model> |
| 5 | * OPENAI_API_KEY=… bun run record.ts --set <name> --journey <id> --provider openai --model <model> |
| 6 | * |
| 7 | * aimock sits between the agent loop and the provider, forwards each request, |
| 8 | * and records the answer. The run is judged by the journey's own checks: a run |
| 9 | * that reached the goal becomes `fixtures/<journey>.<provider>.<model>.json`; |
| 10 | * a run that did not leaves the fixtures alone. Either way the transcript is |
| 11 | * kept in `transcripts/<set>/`, where a set names what was being measured. |
| 12 | * |
| 13 | * One command records one journey with one model, once. There are no retries. |
| 14 | */ |
| 15 | |
| 16 | import { LLMock, type RecordProviderKey } from '@copilotkit/aimock'; |
| 17 | import { execFileSync } from 'node:child_process'; |
| 18 | import { createHash } from 'node:crypto'; |
| 19 | import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; |
| 20 | import { tmpdir } from 'node:os'; |
| 21 | import { join } from 'node:path'; |
| 22 | import { parseArgs } from 'node:util'; |
| 23 | import { z } from 'zod'; |
| 24 | import { adapterFor } from './src/adapters'; |
| 25 | import { fixturePath, HAND_AUTHORED, type Provider } from './src/fixtures'; |
| 26 | import { journeyById, JOURNEYS } from './src/journeys'; |
| 27 | import { runJourney } from './src/run-journey'; |
| 28 | import { buildSkill } from './src/skill'; |
| 29 | |
| 30 | const UPSTREAMS: Record<Provider, { url: string; keyVariable: string }> = { |
| 31 | anthropic: { url: 'https://api.anthropic.com', keyVariable: 'ANTHROPIC_API_KEY' }, |
| 32 | openai: { url: 'https://api.openai.com', keyVariable: 'OPENAI_API_KEY' }, |
| 33 | }; |
| 34 | |
| 35 | const usage = () => |
| 36 | [ |
| 37 | 'Usage: bun run record.ts --set <name> --journey <id> --provider <anthropic|openai> --model <model>', |
| 38 | `Journeys: ${JOURNEYS.map(journey => journey.id).join(', ')}`, |
| 39 | ].join('\n'); |
| 40 | |
| 41 | const { values } = parseArgs({ |
| 42 | options: { |
| 43 | journey: { type: 'string' }, |
| 44 | provider: { type: 'string' }, |
| 45 | model: { type: 'string' }, |
| 46 | set: { type: 'string' }, |
| 47 | // For rehearsing the recorder against a local stand-in for the provider. |
| 48 | upstream: { type: 'string' }, |
| 49 | }, |
| 50 | }); |
| 51 | |
| 52 | const provider = z.enum(['anthropic', 'openai']).safeParse(values.provider); |
| 53 | const set = z |
| 54 | .string() |
| 55 | .regex(/^[a-z0-9][a-z0-9-]*$/) |
| 56 | .safeParse(values.set); |
| 57 | if ( |
| 58 | !set.success || |
| 59 | !values.journey || |
| 60 | !provider.success || |
| 61 | !values.model || |
| 62 | values.model === HAND_AUTHORED |
| 63 | ) { |
| 64 | console.error(usage()); |
| 65 | process.exit(2); |
| 66 | } |
| 67 | const journey = journeyById(values.journey); |
| 68 | const model = values.model; |
| 69 | const upstream = UPSTREAMS[provider.data]; |
| 70 | const apiKey = process.env[upstream.keyVariable]; |
| 71 | if (!apiKey && !values.upstream) { |
| 72 | console.error(`${upstream.keyVariable} is not set; nothing was sent.`); |
| 73 | process.exit(2); |
| 74 | } |
| 75 | |
| 76 | /** What aimock's recorder writes: one file per answered request. */ |
| 77 | const RecordedFile = z.object({ |
| 78 | fixtures: z |
| 79 | .array( |
| 80 | z.looseObject({ |
| 81 | match: z.looseObject({ turnIndex: z.number().optional() }), |
| 82 | response: z.looseObject({ |
| 83 | toolCalls: z.array(z.looseObject({ id: z.string().optional() })).optional(), |
| 84 | }), |
| 85 | }) |
| 86 | ) |
| 87 | .min(1), |
| 88 | }); |
| 89 | |
| 90 | const outDirectory = join(import.meta.dir, 'transcripts', set.data); |
| 91 | mkdirSync(outDirectory, { recursive: true }); |
| 92 | |
| 93 | const git = (...args: string[]): string => |
| 94 | execFileSync('git', args, { cwd: import.meta.dir, encoding: 'utf8' }).trim(); |
| 95 | const CLI_PACKAGE = ':/ts/packages/cli'; |
| 96 | /** What the run measured: the checkout, and the skill text the model was given. */ |
| 97 | const provenance = { |
| 98 | recordedOn: new Date().toISOString().slice(0, 10), |
| 99 | revision: git('rev-parse', '--short', 'HEAD'), |
| 100 | // Uncommitted changes to the CLI or the skill mean the revision alone does not name what ran. |
| 101 | cliPackageModified: git('status', '--porcelain', '--', CLI_PACKAGE) !== '', |
| 102 | // The system prompt and the reference files the model can open. |
| 103 | skillSha256: createHash('sha256').update(JSON.stringify(buildSkill())).digest('hex'), |
| 104 | }; |
| 105 | const rawDirectory = mkdtempSync(join(tmpdir(), 'composio-agent-journey-recording-')); |
| 106 | |
| 107 | const providerKey: RecordProviderKey = provider.data; |
| 108 | const mock = new LLMock({ |
| 109 | port: 0, |
| 110 | logLevel: 'silent', |
| 111 | record: { |
| 112 | providers: { [providerKey]: values.upstream ?? upstream.url }, |
| 113 | // The agent loop only ever sends a placeholder; aimock holds the real key. |
| 114 | providerKeys: apiKey ? { [providerKey]: apiKey } : {}, |
| 115 | fixturePath: rawDirectory, |
| 116 | }, |
| 117 | }); |
| 118 | |
| 119 | const baseURL = await mock.start(); |
| 120 | let exitCode = 0; |
| 121 | try { |
| 122 | const run = await runJourney(journey, adapterFor(provider.data, model, baseURL)); |
| 123 | try { |
| 124 | const checks = journey.checks(run); |
| 125 | const reachedGoal = checks.every(check => check.ok); |
| 126 | // Reported, never a failure: what the model tried on the way. |
| 127 | const detours = journey.detours?.(run) ?? []; |
| 128 | |
| 129 | const recorded = readdirSync(rawDirectory, { recursive: true, encoding: 'utf8' }) |
| 130 | .filter(name => name.endsWith('.json')) |
| 131 | .flatMap( |
| 132 | name => |
| 133 | RecordedFile.parse(JSON.parse(readFileSync(join(rawDirectory, name), 'utf8'))).fixtures |
| 134 | ) |
| 135 | .sort((left, right) => (left.match.turnIndex ?? 0) - (right.match.turnIndex ?? 0)); |
| 136 | |
| 137 | // The recorder keys a turn on its position only. Each turn after the first |
| 138 | // is also tied to the call it answers and to that command's exit code, so a |
| 139 | // CLI that now exits differently leaves the replay without a match. |
| 140 | let commandsSeen = 0; |
| 141 | const fixtures = recorded.map((fixture, index) => { |
| 142 | const previous = index === 0 ? undefined : recorded[index - 1]; |
| 143 | const previousCalls = previous?.response.toolCalls ?? []; |
| 144 | commandsSeen += previousCalls.length; |
| 145 | const answered = run.transcript.commands[commandsSeen - 1]; |
| 146 | const lastCallId = previousCalls.at(-1)?.id; |
| 147 | return previous && lastCallId && answered |
| 148 | ? { |
| 149 | ...fixture, |
| 150 | match: { |
| 151 | ...fixture.match, |
| 152 | toolCallId: lastCallId, |
| 153 | toolResultContains: `"exit_code":${answered.exitCode},`, |
| 154 | }, |
| 155 | } |
| 156 | : fixture; |
| 157 | }); |
| 158 | |
| 159 | const fixtureFile = JSON.stringify( |
| 160 | { |
| 161 | $comment: `Recorded from ${provider.data} ${model} by record.ts. Do not edit by hand; record again.`, |
| 162 | fixtures, |
| 163 | }, |
| 164 | null, |
| 165 | 2 |
| 166 | ); |
| 167 | const transcriptFile = JSON.stringify( |
| 168 | { |
| 169 | journey: journey.id, |
| 170 | provider: provider.data, |
| 171 | model, |
| 172 | ...provenance, |
| 173 | prompt: journey.prompt, |
| 174 | reachedGoal, |
| 175 | stop: run.transcript.stop, |
| 176 | modelTurns: run.transcript.modelTurns, |
| 177 | checks, |
| 178 | detours, |
| 179 | commands: run.transcript.commands.map(({ command, exitCode: code, stdout, stderr }) => ({ |
| 180 | command, |
| 181 | exitCode: code, |
| 182 | stdout, |
| 183 | stderr, |
| 184 | })), |
| 185 | messages: run.transcript.messages, |
| 186 | replies: run.transcript.replies, |
| 187 | }, |
| 188 | null, |
| 189 | 2 |
| 190 | ); |
| 191 | |
| 192 | // Nothing that could carry a credential is kept, whatever the recorder did. |
| 193 | const leaks = [fixtureFile, transcriptFile].flatMap(text => [ |
| 194 | ...(apiKey && text.includes(apiKey) ? ['the API key'] : []), |
| 195 | // A provider key by its prefix, or a credential header recorded as a JSON key. |
| 196 | ...(/sk-ant-|sk-proj-|"(?:x-api-key|authorization)"\s*:/i.test(text) |
| 197 | ? ['a credential marker'] |
| 198 | : []), |
| 199 | ]); |
| 200 | if (leaks.length > 0) { |
| 201 | throw new Error( |
| 202 | `Refusing to write: the recording contains ${[...new Set(leaks)].join(' and ')}.` |
| 203 | ); |
| 204 | } |
| 205 | |
| 206 | const stem = `${journey.id}.${provider.data}.${model}`; |
| 207 | const transcriptPath = join(outDirectory, `${stem}.json`); |
| 208 | writeFileSync(transcriptPath, `${transcriptFile}\n`); |
| 209 | |
| 210 | console.log(`journey ${journey.id}`); |
| 211 | console.log(`model ${provider.data} ${model}`); |
| 212 | console.log( |
| 213 | `stop ${JSON.stringify(run.transcript.stop)} after ${run.transcript.modelTurns} model turn(s)` |
| 214 | ); |
| 215 | for (const command of run.transcript.commands) { |
| 216 | console.log(` [${command.exitCode}] ${command.command}`); |
| 217 | } |
| 218 | for (const check of checks) { |
| 219 | console.log( |
| 220 | ` ${check.ok ? 'ok ' : 'FAIL'} ${check.name}${check.ok ? '' : ` (saw ${check.seen ?? 'nothing'})`}` |
| 221 | ); |
| 222 | } |
| 223 | for (const detour of detours.filter(taken => !taken.ok)) { |
| 224 | console.log(` detour ${detour.name} (saw ${detour.seen ?? 'it'})`); |
| 225 | } |
| 226 | console.log(`transcript ${transcriptPath}`); |
| 227 | |
| 228 | if (reachedGoal) { |
| 229 | const target = fixturePath(journey.id, provider.data, model); |
| 230 | writeFileSync(target, `${fixtureFile}\n`); |
| 231 | console.log(`fixture ${target} (${fixtures.length} turn(s))`); |
| 232 | } else { |
| 233 | console.log('fixture not written: the run did not reach the goal'); |
| 234 | exitCode = 1; |
| 235 | } |
| 236 | } finally { |
| 237 | await run.rig.stop(); |
| 238 | } |
| 239 | } finally { |
| 240 | await mock.stop(); |
| 241 | rmSync(rawDirectory, { recursive: true, force: true }); |
| 242 | } |
| 243 | process.exit(exitCode); |