Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 539 lines 20.1 KB
Raw
1 /**
2 * The agent journeys: a user prompt, the state the CLI starts in, and what
3 * must be true afterwards.
4 *
5 * A check reads the commands that ran, what they printed, and what the fake
6 * backend received. None of them reads the model's prose except to see that it
7 * stopped: a model can say "done" after every command failed.
8 */
9
10 import {
11 CONSUMER_PROJECT,
12 CONSUMER_USER_ID,
13 DEVELOPER_PROJECT,
14 GLOBAL_TEST_USER_ID,
15 MEMBER,
16 ORG,
17 type JournalEntry,
18 type World,
19 } from '@e2e-tests/cli-fake-backend';
20 import type { CliRig } from '@e2e-tests/cli-fake-backend/rig';
21 import { z } from 'zod';
22 import type { ExecutedCommand, ShellOutcome, Transcript } from './agent';
23 import type { Usage } from './usage';
24
25 /** What a journey is graded on: what ran, what it printed, and what it changed. */
26 export interface Evidence {
27 readonly transcript: Transcript;
28 /** The fake's state once the model had stopped. */
29 readonly world: World;
30 /** What the fake received between the first and the last model turn. */
31 readonly requests: ReadonlyArray<JournalEntry>;
32 /** What the fake received while each of `transcript.commands` ran, in the same order. */
33 readonly commandRequests: ReadonlyArray<ReadonlyArray<JournalEntry>>;
34 /** Ids of the connections the journey's setup left behind, before the first model turn. */
35 readonly startingConnections: ReadonlyArray<string>;
36 /**
37 * `composio whoami` in a new process, once the model had stopped: what the
38 * next command on this machine would find. Its requests are not in `requests`.
39 */
40 readonly whoamiAfter: ShellOutcome;
41 }
42
43 export interface JourneyRun extends Evidence {
44 readonly rig: CliRig;
45 /** `worldHash` of the fake once the setup had run, before the first model turn. */
46 readonly startingWorldHash: string;
47 /** What the run cost, when the agent reports it. */
48 readonly usage?: Usage;
49 }
50
51 export interface Check {
52 readonly name: string;
53 readonly ok: boolean;
54 /** What was seen instead, for the failure message. */
55 readonly seen?: string;
56 }
57
58 export interface Journey {
59 readonly id: string;
60 readonly title: string;
61 readonly prompt: string;
62 /**
63 * `train` journeys are the ones a skill change is tuned against; `holdout`
64 * journeys only measure, so a change that fits the train set and nothing
65 * else shows up there. See HILLCLIMB.md.
66 */
67 readonly split: 'train' | 'holdout';
68 /** Directory the agent's shell starts in; defaults to the home directory. */
69 readonly cwd?: string;
70 readonly setup: (rig: CliRig) => Promise<void>;
71 /**
72 * What must hold however the model got there. These grade what the run
73 * changed: what was executed, connected, and stored. A run that reached the
74 * goal after trying the wrong thing still passes.
75 */
76 readonly checks: (run: Evidence) => ReadonlyArray<Check>;
77 /**
78 * What a confused agent does on the way: a command on the other tree, a
79 * request to the other project. `ok: false` means the run took the detour.
80 * Reported next to the checks and never a failure, so a wrong-tree rate can
81 * be read separately from the success rate.
82 */
83 readonly detours?: (run: Evidence) => ReadonlyArray<Check>;
84 /**
85 * What must hold on the hand-authored path only: how the CLI answered the
86 * specific commands that path scripts.
87 */
88 readonly scriptedChecks?: (run: Evidence) => ReadonlyArray<Check>;
89 }
90
91 export const DEV_DIRECTORY = '/work/app';
92
93 const check = (name: string, ok: boolean, seen?: unknown): Check => ({
94 name,
95 ok,
96 seen:
97 ok || seen === undefined ? undefined : typeof seen === 'string' ? seen : JSON.stringify(seen),
98 });
99
100 const isComposio = (command: ExecutedCommand, pattern: RegExp) => pattern.test(command.command);
101 // Where a `composio` invocation can start in a command line: at its start,
102 // after a separator, or after the directory of a binary named by its path.
103 const DEV_TREE = /(^|[;&|(\s/])composio\s+dev(\s|$)/;
104 const ROOT_EXECUTE = /(^|[;&|(\s/])composio\s+execute\s/;
105 /** A root `composio execute` that runs the tool, not one that only describes it. */
106 const ROOT_EXECUTE_RUNS =
107 /(^|[;&|(\s/])composio\s+execute\s(?![^;&|]*--(?:get-schema|dry-run|help)\b)/;
108 const ROOT_LINK = /(^|[;&|(\s/])composio\s+link(\s|$)/;
109 const RUN = /(^|[;&|(\s/])composio\s+run\s/g;
110 const PLAYGROUND_EXECUTE = /composio\s+dev\s+(?:--\S+\s+\S+\s+)*playground-execute\s/;
111
112 const SEND = 'SLACK_SEND_MESSAGE';
113
114 /** Every tool execution that completed, whichever route ran it. */
115 const executions = (requests: ReadonlyArray<JournalEntry>) =>
116 requests.flatMap(entry =>
117 // A Dashboard execution answers 200 even when the tool was refused; the effect says it ran.
118 entry.effect === 'tool-executed' && entry.tool ? [{ entry, tool: entry.tool }] : []
119 );
120
121 const describeExecutions = (requests: ReadonlyArray<JournalEntry>) =>
122 executions(requests).map(({ entry, tool }) => ({
123 tool: tool.slug,
124 project: entry.project,
125 user: entry.userId,
126 arguments: tool.arguments,
127 }));
128
129 /** `general` and `#general` name the same channel. */
130 const channelName = (value: unknown): string =>
131 typeof value === 'string' ? value.trim().replace(/^#/, '') : '';
132
133 /**
134 * The person asked for one message. So one `SLACK_SEND_MESSAGE` completed in
135 * the whole run, on any project and through any route, and it is that
136 * message: a second send, a send on the other project, another channel, and
137 * other words each fail a check of their own. A lookup beside it changes
138 * nothing and is let through; any other tool that changes something is not.
139 */
140 const sentOnce = (
141 run: Evidence,
142 expected: {
143 text: string;
144 project: string;
145 userId: string;
146 /**
147 * Whether the request itself must name the project. A developer command
148 * sends it; a consumer execution names none and the Dashboard resolves it.
149 */
150 scope: 'named' | 'resolved';
151 }
152 ): ReadonlyArray<Check> => {
153 const all = executions(run.requests);
154 const sends = all.filter(({ tool }) => tool.slug === SEND);
155 const otherChanges = all.filter(
156 ({ tool }) => tool.slug !== SEND && run.world.findTool(tool.slug)?.readOnly !== true
157 );
158 const seen = describeExecutions(run.requests);
159 const everySend = (holds: (execution: (typeof sends)[number]) => boolean) =>
160 sends.length > 0 && sends.every(holds);
161 return [
162 check(`${SEND} completed once in the whole run`, sends.length === 1, seen),
163 check(
164 `it ran on the ${expected.project} project, as ${expected.userId}`,
165 everySend(
166 ({ entry }) =>
167 entry.project === expected.project &&
168 entry.userId === expected.userId &&
169 (entry.requestedProject === expected.project ||
170 (expected.scope === 'resolved' && entry.requestedProject === null))
171 ),
172 seen
173 ),
174 check(
175 'it went to #general',
176 everySend(({ tool }) => channelName(tool.arguments.channel) === 'general'),
177 seen
178 ),
179 check(
180 `its text was exactly "${expected.text}"`,
181 everySend(
182 ({ tool }) =>
183 typeof tool.arguments.text === 'string' && tool.arguments.text.trim() === expected.text
184 ),
185 seen
186 ),
187 check('no other tool that changes something ran', otherChanges.length === 0, seen),
188 ];
189 };
190
191 /** A connection made during the run on a project the task never involved. */
192 const connectedOnlyOn = (run: Evidence, project: string): Check => {
193 const stray = run.world.connections.filter(
194 connection =>
195 connection.status === 'ACTIVE' &&
196 connection.projectLabel !== project &&
197 !run.startingConnections.includes(connection.id)
198 );
199 return check(
200 `no account was connected outside the ${project} project`,
201 stray.length === 0,
202 stray.map(connection => `${connection.toolkit} on ${connection.projectLabel}`)
203 );
204 };
205
206 const ranNo = (run: Evidence, name: string, pattern: RegExp): Check => {
207 const matching = run.transcript.commands.filter(command => isComposio(command, pattern));
208 return check(
209 name,
210 matching.length === 0,
211 matching.map(command => command.command)
212 );
213 };
214
215 /** What `composio whoami` prints on stdout off a terminal, as far as a login is concerned. */
216 const Whoami = z.looseObject({
217 account_type: z.string(),
218 email: z.string(),
219 current_org_name: z.string(),
220 });
221
222 /** The login a `composio whoami` reported, or `undefined` when it reported none. */
223 const loginReportedBy = (outcome: ShellOutcome) => {
224 if (outcome.exitCode !== 0) return undefined;
225 try {
226 return Whoami.safeParse(JSON.parse(outcome.stdout)).data;
227 } catch {
228 return undefined;
229 }
230 };
231
232 const lastCommandSucceeded = (run: Evidence, pattern: RegExp): boolean => {
233 const matching = run.transcript.commands.filter(command => isComposio(command, pattern));
234 return matching.at(-1)?.exitCode === 0;
235 };
236
237 const stoppedByItself = (run: Evidence): Check =>
238 check(
239 'the model finished by itself, within the turn limit',
240 run.transcript.stop === 'finished',
241 run.transcript.stop
242 );
243
244 const loginRig = async (rig: CliRig): Promise<void> => {
245 await rig.login();
246 };
247
248 const developerRig = async (rig: CliRig): Promise<void> => {
249 await rig.login();
250 const result = await rig.run(
251 `mkdir -p ${DEV_DIRECTORY} && cd ${DEV_DIRECTORY} && composio dev init --yes --no-browser`
252 );
253 if (result.exitCode !== 0) throw new Error(`composio dev init failed: ${result.stderr}`);
254 };
255
256 /**
257 * Requests that acted on a project other than the one the task is about.
258 * Organization routes (`_internal.org.*`) are left out: the CLI reads the
259 * org's settings on its own, under Claude Code on every command, and the
260 * project the fake journals for them is the key's default, not a choice.
261 */
262 const requestsOutside = (run: Evidence, project: string): Check => {
263 const outside = run.requests.filter(
264 entry =>
265 (entry.role === 'backend' || entry.role === 'dashboard') &&
266 !entry.route?.startsWith('_internal.org.') &&
267 entry.project !== null &&
268 entry.project !== project
269 );
270 return check(
271 `every request that named a project named the ${project} project`,
272 outside.length === 0,
273 [...new Set(outside.map(entry => `${entry.route} on ${entry.project}`))]
274 );
275 };
276
277 /** What a developer task looks like when the agent reaches for the root tree. */
278 const rootTreeDetours = (run: Evidence): ReadonlyArray<Check> => [
279 ranNo(run, 'no root `composio execute` tried to run the tool', ROOT_EXECUTE_RUNS),
280 ranNo(run, 'no root `composio link` ran', ROOT_LINK),
281 check(
282 'nothing was executed or linked on the consumer project, completed or not',
283 !run.requests.some(
284 entry =>
285 entry.project === CONSUMER_PROJECT.label &&
286 (entry.route === 'dashboard.execute' ||
287 entry.route === 'toolRouter.session.execute' ||
288 entry.route === 'toolRouter.session.link')
289 )
290 ),
291 ];
292
293 /** What a consumer task looks like when the agent reaches for the dev tree. */
294 const devTreeDetours = (run: Evidence): ReadonlyArray<Check> => [
295 ranNo(run, 'no `composio dev` command ran', DEV_TREE),
296 requestsOutside(run, CONSUMER_PROJECT.label),
297 ];
298
299 export const JOURNEYS: ReadonlyArray<Journey> = [
300 {
301 id: 'consumer-slack-message',
302 title: 'consumer task: send a Slack message from an unconnected account',
303 prompt: 'Send a Slack message to the #general channel that says: deploy finished',
304 split: 'train',
305 setup: loginRig,
306 checks: run => [
307 stoppedByItself(run),
308 check(
309 'the last `composio execute` exited 0',
310 lastCommandSucceeded(run, ROOT_EXECUTE),
311 run.transcript.commands.map(command => `${command.exitCode} ${command.command}`)
312 ),
313 ...sentOnce(run, {
314 text: 'deploy finished',
315 project: CONSUMER_PROJECT.label,
316 userId: CONSUMER_USER_ID,
317 scope: 'resolved',
318 }),
319 check(
320 'the account was linked on the consumer project',
321 run.world.activeConnection(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack') !== undefined
322 ),
323 connectedOnlyOn(run, CONSUMER_PROJECT.label),
324 ],
325 detours: devTreeDetours,
326 },
327 {
328 id: 'developer-auth-configs-and-execute',
329 title: 'developer task: list auth configs and run a tool as the test user',
330 prompt:
331 'I am building an app on Composio. In my developer project, list the auth configs, then execute SLACK_SEND_MESSAGE as my test user to post "hello from dev" in #general.',
332 split: 'holdout',
333 cwd: DEV_DIRECTORY,
334 setup: async rig => {
335 await developerRig(rig);
336 rig.fake.world.connect(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack');
337 },
338 checks: run => [
339 stoppedByItself(run),
340 check(
341 '`composio dev auth-configs list` ran and exited 0',
342 lastCommandSucceeded(run, /composio\s+dev\s+auth-configs\s+list/),
343 run.transcript.commands.map(command => `${command.exitCode} ${command.command}`)
344 ),
345 check(
346 'the auth configs were read from the developer project',
347 run.requests.some(
348 entry =>
349 entry.route === 'authConfigs.list' &&
350 entry.status === 200 &&
351 entry.project === DEVELOPER_PROJECT.label
352 )
353 ),
354 ...sentOnce(run, {
355 text: 'hello from dev',
356 project: DEVELOPER_PROJECT.label,
357 userId: GLOBAL_TEST_USER_ID,
358 scope: 'named',
359 }),
360 check(
361 'it ran through `composio dev playground-execute`',
362 lastCommandSucceeded(run, PLAYGROUND_EXECUTE)
363 ),
364 connectedOnlyOn(run, DEVELOPER_PROJECT.label),
365 ],
366 detours: rootTreeDetours,
367 },
368 {
369 id: 'developer-execute-unconnected',
370 title: 'tree confusion: a developer task that needs a connection first',
371 prompt:
372 'In my Composio developer project, execute SLACK_SEND_MESSAGE as my test user to post "hello from dev" in #general.',
373 split: 'train',
374 cwd: DEV_DIRECTORY,
375 setup: developerRig,
376 checks: run => [
377 stoppedByItself(run),
378 ...sentOnce(run, {
379 text: 'hello from dev',
380 project: DEVELOPER_PROJECT.label,
381 userId: GLOBAL_TEST_USER_ID,
382 scope: 'named',
383 }),
384 check(
385 'the account was linked on the developer project for the test user',
386 run.world.activeConnection(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack') !==
387 undefined
388 ),
389 // The confusion this journey exists for: `composio link` connects the
390 // consumer user, the person approves it, and the developer task is no nearer.
391 connectedOnlyOn(run, DEVELOPER_PROJECT.label),
392 check(
393 'the last `composio dev playground-execute` exited 0',
394 lastCommandSucceeded(run, PLAYGROUND_EXECUTE),
395 run.transcript.commands.map(command => `${command.exitCode} ${command.command}`)
396 ),
397 ],
398 detours: rootTreeDetours,
399 scriptedChecks: run => {
400 const [wrongTree, firstAttempt] = run.transcript.commands;
401 return [
402 // A command from the root tree's vocabulary on the dev tree: recoverable.
403 check(
404 '`composio dev execute` exits 1 with nothing on stdout',
405 wrongTree?.exitCode === 1 && wrongTree.stdout === '',
406 wrongTree
407 ),
408 check(
409 'its stderr names the unknown subcommand',
410 wrongTree?.stderr.includes('Unknown subcommand "execute" for "composio dev"') === true,
411 wrongTree?.stderr.slice(-300)
412 ),
413 check(
414 'the help printed with it lists the real command, and both survive truncation',
415 wrongTree?.payload.includes('playground-execute') === true &&
416 wrongTree.payload.includes('Unknown subcommand'),
417 wrongTree?.payload.length
418 ),
419 // The not-connected message names the link of the tree it was raised
420 // on. A root-tree link would connect the consumer user and fix nothing.
421 check(
422 'the unconnected `playground-execute` names the dev-tree link',
423 firstAttempt?.exitCode === 1 &&
424 firstAttempt.stdout.includes('Run `composio dev connected-accounts link slack`') &&
425 !firstAttempt.stdout.includes('`composio link slack`'),
426 firstAttempt?.stdout
427 ),
428 ];
429 },
430 },
431 {
432 id: 'first-run-login',
433 title: 'first run: find out the CLI is not logged in, and log in',
434 prompt: 'Check whether the Composio CLI on this machine is logged in. If it is not, log in.',
435 split: 'train',
436 setup: async () => {},
437 checks: run => {
438 const whoami = run.transcript.commands.filter(command =>
439 isComposio(command, /composio\s+whoami/)
440 );
441 const stored = loginReportedBy(run.whoamiAfter);
442 return [
443 stoppedByItself(run),
444 // By what it printed as well as by its exit code: an agent that pipes
445 // `whoami` into `head` sees the message with exit code 0.
446 check(
447 'the first `composio whoami` reported no login',
448 whoami[0] !== undefined &&
449 (whoami[0].exitCode === 1 ||
450 /not logged in/i.test(`${whoami[0].stdout}\n${whoami[0].stderr}`)),
451 whoami.map(command => `${command.exitCode} ${command.stdout.slice(0, 120)}`)
452 ),
453 check(
454 'exactly one CLI login session was created, and it was linked',
455 run.world.loginSessions.length === 1 && run.world.loginSessions[0]?.status === 'linked',
456 run.world.loginSessions
457 ),
458 check(
459 'the login completed: the user API key was fetched for the linked session',
460 run.requests.some(
461 entry => entry.route === '_internal.cli.getSession' && entry.status === 200
462 ) &&
463 run.requests.some(
464 entry => entry.route === 'auth.session.retrieveInfo' && entry.keyKnown
465 )
466 ),
467 // The requests above happen in the process that logged in. This is the
468 // next process: it finds the login only if it was stored.
469 check(
470 'the login was stored: a new `composio whoami` exits 0 as the person, in their org',
471 stored?.account_type === 'human' &&
472 stored.email === MEMBER.email &&
473 stored.current_org_name === ORG.name,
474 run.whoamiAfter
475 ),
476 check(
477 'no agent account was signed up in place of the person',
478 !run.requests.some(entry => entry.route === 'agents.signup')
479 ),
480 ];
481 },
482 },
483 {
484 id: 'run-inline-script',
485 title: '`composio run`: one inline script that executes a tool',
486 prompt:
487 'Use a single `composio run` inline script to send "build passed" to #general on Slack and print the timestamp of the sent message.',
488 split: 'holdout',
489 setup: async rig => {
490 await rig.login();
491 rig.fake.world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
492 },
493 checks: run => {
494 // The commands during which a tool execution completed.
495 const sending = run.transcript.commands.filter(
496 (_, index) => executions(run.commandRequests[index] ?? []).length > 0
497 );
498 return [
499 stoppedByItself(run),
500 ...sentOnce(run, {
501 text: 'build passed',
502 project: CONSUMER_PROJECT.label,
503 userId: CONSUMER_USER_ID,
504 scope: 'resolved',
505 }),
506 // The send and the timestamp belong to one command, and that command
507 // is the script: not `composio execute` followed by a script that
508 // prints what it was told. This reads the command's text, so it holds
509 // for the ways a shell line names `composio`, not for an alias or a
510 // wrapper script.
511 check(
512 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp',
513 sending.length === 1 &&
514 sending.every(
515 command =>
516 (command.command.match(RUN) ?? []).length === 1 &&
517 !isComposio(command, ROOT_EXECUTE_RUNS) &&
518 !isComposio(command, PLAYGROUND_EXECUTE) &&
519 command.exitCode === 0 &&
520 command.stdout.includes('1767225600.000100')
521 ),
522 sending.map(command => `${command.exitCode} ${command.command.slice(0, 160)}`)
523 ),
524 connectedOnlyOn(run, CONSUMER_PROJECT.label),
525 ];
526 },
527 detours: devTreeDetours,
528 },
529 ];
530
531 export const journeyById = (id: string): Journey => {
532 const journey = JOURNEYS.find(candidate => candidate.id === id);
533 if (!journey) {
534 throw new Error(
535 `Unknown journey "${id}". Known: ${JOURNEYS.map(candidate => candidate.id).join(', ')}`
536 );
537 }
538 return journey;
539 };