@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 243 lines 8.71 KB
Raw
1 /**
2 * Records one journey with a real model.
3 *
4 * ANTHROPIC_API_KEY=… bun run record.ts --set <name> --journey <id> --provider anthropic --model <model>
5 * OPENAI_API_KEY=… bun run record.ts --set <name> --journey <id> --provider openai --model <model>
6 *
7 * aimock sits between the agent loop and the provider, forwards each request,
8 * and records the answer. The run is judged by the journey's own checks: a run
9 * that reached the goal becomes `fixtures/<journey>.<provider>.<model>.json`;
10 * a run that did not leaves the fixtures alone. Either way the transcript is
11 * kept in `transcripts/<set>/`, where a set names what was being measured.
12 *
13 * One command records one journey with one model, once. There are no retries.
14 */
15
16 import { LLMock, type RecordProviderKey } from '@copilotkit/aimock';
17 import { execFileSync } from 'node:child_process';
18 import { createHash } from 'node:crypto';
19 import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
20 import { tmpdir } from 'node:os';
21 import { join } from 'node:path';
22 import { parseArgs } from 'node:util';
23 import { z } from 'zod';
24 import { adapterFor } from './src/adapters';
25 import { fixturePath, HAND_AUTHORED, type Provider } from './src/fixtures';
26 import { journeyById, JOURNEYS } from './src/journeys';
27 import { runJourney } from './src/run-journey';
28 import { buildSkill } from './src/skill';
29
30 const UPSTREAMS: Record<Provider, { url: string; keyVariable: string }> = {
31 anthropic: { url: 'https://api.anthropic.com', keyVariable: 'ANTHROPIC_API_KEY' },
32 openai: { url: 'https://api.openai.com', keyVariable: 'OPENAI_API_KEY' },
33 };
34
35 const usage = () =>
36 [
37 'Usage: bun run record.ts --set <name> --journey <id> --provider <anthropic|openai> --model <model>',
38 `Journeys: ${JOURNEYS.map(journey => journey.id).join(', ')}`,
39 ].join('\n');
40
41 const { values } = parseArgs({
42 options: {
43 journey: { type: 'string' },
44 provider: { type: 'string' },
45 model: { type: 'string' },
46 set: { type: 'string' },
47 // For rehearsing the recorder against a local stand-in for the provider.
48 upstream: { type: 'string' },
49 },
50 });
51
52 const provider = z.enum(['anthropic', 'openai']).safeParse(values.provider);
53 const set = z
54 .string()
55 .regex(/^[a-z0-9][a-z0-9-]*$/)
56 .safeParse(values.set);
57 if (
58 !set.success ||
59 !values.journey ||
60 !provider.success ||
61 !values.model ||
62 values.model === HAND_AUTHORED
63 ) {
64 console.error(usage());
65 process.exit(2);
66 }
67 const journey = journeyById(values.journey);
68 const model = values.model;
69 const upstream = UPSTREAMS[provider.data];
70 const apiKey = process.env[upstream.keyVariable];
71 if (!apiKey && !values.upstream) {
72 console.error(`${upstream.keyVariable} is not set; nothing was sent.`);
73 process.exit(2);
74 }
75
76 /** What aimock's recorder writes: one file per answered request. */
77 const RecordedFile = z.object({
78 fixtures: z
79 .array(
80 z.looseObject({
81 match: z.looseObject({ turnIndex: z.number().optional() }),
82 response: z.looseObject({
83 toolCalls: z.array(z.looseObject({ id: z.string().optional() })).optional(),
84 }),
85 })
86 )
87 .min(1),
88 });
89
90 const outDirectory = join(import.meta.dir, 'transcripts', set.data);
91 mkdirSync(outDirectory, { recursive: true });
92
93 const git = (...args: string[]): string =>
94 execFileSync('git', args, { cwd: import.meta.dir, encoding: 'utf8' }).trim();
95 const CLI_PACKAGE = ':/ts/packages/cli';
96 /** What the run measured: the checkout, and the skill text the model was given. */
97 const provenance = {
98 recordedOn: new Date().toISOString().slice(0, 10),
99 revision: git('rev-parse', '--short', 'HEAD'),
100 // Uncommitted changes to the CLI or the skill mean the revision alone does not name what ran.
101 cliPackageModified: git('status', '--porcelain', '--', CLI_PACKAGE) !== '',
102 // The system prompt and the reference files the model can open.
103 skillSha256: createHash('sha256').update(JSON.stringify(buildSkill())).digest('hex'),
104 };
105 const rawDirectory = mkdtempSync(join(tmpdir(), 'composio-agent-journey-recording-'));
106
107 const providerKey: RecordProviderKey = provider.data;
108 const mock = new LLMock({
109 port: 0,
110 logLevel: 'silent',
111 record: {
112 providers: { [providerKey]: values.upstream ?? upstream.url },
113 // The agent loop only ever sends a placeholder; aimock holds the real key.
114 providerKeys: apiKey ? { [providerKey]: apiKey } : {},
115 fixturePath: rawDirectory,
116 },
117 });
118
119 const baseURL = await mock.start();
120 let exitCode = 0;
121 try {
122 const run = await runJourney(journey, adapterFor(provider.data, model, baseURL));
123 try {
124 const checks = journey.checks(run);
125 const reachedGoal = checks.every(check => check.ok);
126 // Reported, never a failure: what the model tried on the way.
127 const detours = journey.detours?.(run) ?? [];
128
129 const recorded = readdirSync(rawDirectory, { recursive: true, encoding: 'utf8' })
130 .filter(name => name.endsWith('.json'))
131 .flatMap(
132 name =>
133 RecordedFile.parse(JSON.parse(readFileSync(join(rawDirectory, name), 'utf8'))).fixtures
134 )
135 .sort((left, right) => (left.match.turnIndex ?? 0) - (right.match.turnIndex ?? 0));
136
137 // The recorder keys a turn on its position only. Each turn after the first
138 // is also tied to the call it answers and to that command's exit code, so a
139 // CLI that now exits differently leaves the replay without a match.
140 let commandsSeen = 0;
141 const fixtures = recorded.map((fixture, index) => {
142 const previous = index === 0 ? undefined : recorded[index - 1];
143 const previousCalls = previous?.response.toolCalls ?? [];
144 commandsSeen += previousCalls.length;
145 const answered = run.transcript.commands[commandsSeen - 1];
146 const lastCallId = previousCalls.at(-1)?.id;
147 return previous && lastCallId && answered
148 ? {
149 ...fixture,
150 match: {
151 ...fixture.match,
152 toolCallId: lastCallId,
153 toolResultContains: `"exit_code":${answered.exitCode},`,
154 },
155 }
156 : fixture;
157 });
158
159 const fixtureFile = JSON.stringify(
160 {
161 $comment: `Recorded from ${provider.data} ${model} by record.ts. Do not edit by hand; record again.`,
162 fixtures,
163 },
164 null,
165 2
166 );
167 const transcriptFile = JSON.stringify(
168 {
169 journey: journey.id,
170 provider: provider.data,
171 model,
172 ...provenance,
173 prompt: journey.prompt,
174 reachedGoal,
175 stop: run.transcript.stop,
176 modelTurns: run.transcript.modelTurns,
177 checks,
178 detours,
179 commands: run.transcript.commands.map(({ command, exitCode: code, stdout, stderr }) => ({
180 command,
181 exitCode: code,
182 stdout,
183 stderr,
184 })),
185 messages: run.transcript.messages,
186 replies: run.transcript.replies,
187 },
188 null,
189 2
190 );
191
192 // Nothing that could carry a credential is kept, whatever the recorder did.
193 const leaks = [fixtureFile, transcriptFile].flatMap(text => [
194 ...(apiKey && text.includes(apiKey) ? ['the API key'] : []),
195 // A provider key by its prefix, or a credential header recorded as a JSON key.
196 ...(/sk-ant-|sk-proj-|"(?:x-api-key|authorization)"\s*:/i.test(text)
197 ? ['a credential marker']
198 : []),
199 ]);
200 if (leaks.length > 0) {
201 throw new Error(
202 `Refusing to write: the recording contains ${[...new Set(leaks)].join(' and ')}.`
203 );
204 }
205
206 const stem = `${journey.id}.${provider.data}.${model}`;
207 const transcriptPath = join(outDirectory, `${stem}.json`);
208 writeFileSync(transcriptPath, `${transcriptFile}\n`);
209
210 console.log(`journey ${journey.id}`);
211 console.log(`model ${provider.data} ${model}`);
212 console.log(
213 `stop ${JSON.stringify(run.transcript.stop)} after ${run.transcript.modelTurns} model turn(s)`
214 );
215 for (const command of run.transcript.commands) {
216 console.log(` [${command.exitCode}] ${command.command}`);
217 }
218 for (const check of checks) {
219 console.log(
220 ` ${check.ok ? 'ok ' : 'FAIL'} ${check.name}${check.ok ? '' : ` (saw ${check.seen ?? 'nothing'})`}`
221 );
222 }
223 for (const detour of detours.filter(taken => !taken.ok)) {
224 console.log(` detour ${detour.name} (saw ${detour.seen ?? 'it'})`);
225 }
226 console.log(`transcript ${transcriptPath}`);
227
228 if (reachedGoal) {
229 const target = fixturePath(journey.id, provider.data, model);
230 writeFileSync(target, `${fixtureFile}\n`);
231 console.log(`fixture ${target} (${fixtures.length} turn(s))`);
232 } else {
233 console.log('fixture not written: the run did not reach the goal');
234 exitCode = 1;
235 }
236 } finally {
237 await run.rig.stop();
238 }
239 } finally {
240 await mock.stop();
241 rmSync(rawDirectory, { recursive: true, force: true });
242 }
243 process.exit(exitCode);