1
+/**
2
+ * The agent journeys: a user prompt, the state the CLI starts in, and what
3
+ * must be true afterwards.
4
+ *
5
+ * A check reads the commands that ran, what they printed, and what the fake
6
+ * backend received. None of them reads the model's prose except to see that it
7
+ * stopped: a model can say "done" after every command failed.
8
+ */
9
+
10
+import {
11
+ CONSUMER_PROJECT,
12
+ CONSUMER_USER_ID,
13
+ DEVELOPER_PROJECT,
14
+ GLOBAL_TEST_USER_ID,
15
+ MEMBER,
16
+ ORG,
17
+ type JournalEntry,
18
+ type World,
19
+} from '@e2e-tests/cli-fake-backend';
20
+import type { CliRig } from '@e2e-tests/cli-fake-backend/rig';
21
+import { z } from 'zod';
22
+import type { ExecutedCommand, ShellOutcome, Transcript } from './agent';
23
+import type { Usage } from './usage';
24
+
25
+/** What a journey is graded on: what ran, what it printed, and what it changed. */
26
+export interface Evidence {
27
+ readonly transcript: Transcript;
28
+ /** The fake's state once the model had stopped. */
29
+ readonly world: World;
30
+ /** What the fake received between the first and the last model turn. */
31
+ readonly requests: ReadonlyArray<JournalEntry>;
32
+ /** What the fake received while each of `transcript.commands` ran, in the same order. */
33
+ readonly commandRequests: ReadonlyArray<ReadonlyArray<JournalEntry>>;
34
+ /** Ids of the connections the journey's setup left behind, before the first model turn. */
35
+ readonly startingConnections: ReadonlyArray<string>;
36
+ /**
37
+ * `composio whoami` in a new process, once the model had stopped: what the
38
+ * next command on this machine would find. Its requests are not in `requests`.
39
+ */
40
+ readonly whoamiAfter: ShellOutcome;
41
+}
42
+
43
+export interface JourneyRun extends Evidence {
44
+ readonly rig: CliRig;
45
+ /** What the run cost, when the agent reports it. */
46
+ readonly usage?: Usage;
47
+}
48
+
49
+export interface Check {
50
+ readonly name: string;
51
+ readonly ok: boolean;
52
+ /** What was seen instead, for the failure message. */
53
+ readonly seen?: string;
54
+}
55
+
56
+export interface Journey {
57
+ readonly id: string;
58
+ readonly title: string;
59
+ readonly prompt: string;
60
+ /** Directory the agent's shell starts in; defaults to the home directory. */
61
+ readonly cwd?: string;
62
+ readonly setup: (rig: CliRig) => Promise<void>;
63
+ /**
64
+ * What must hold however the model got there. These grade what the run
65
+ * changed: what was executed, connected, and stored. A run that reached the
66
+ * goal after trying the wrong thing still passes.
67
+ */
68
+ readonly checks: (run: Evidence) => ReadonlyArray<Check>;
69
+ /**
70
+ * What a confused agent does on the way: a command on the other tree, a
71
+ * request to the other project. `ok: false` means the run took the detour.
72
+ * Reported next to the checks and never a failure, so a wrong-tree rate can
73
+ * be read separately from the success rate.
74
+ */
75
+ readonly detours?: (run: Evidence) => ReadonlyArray<Check>;
76
+ /**
77
+ * What must hold on the hand-authored path only: how the CLI answered the
78
+ * specific commands that path scripts.
79
+ */
80
+ readonly scriptedChecks?: (run: Evidence) => ReadonlyArray<Check>;
81
+}
82
+
83
+export const DEV_DIRECTORY = '/work/app';
84
+
85
+const check = (name: string, ok: boolean, seen?: unknown): Check => ({
86
+ name,
87
+ ok,
88
+ seen:
89
+ ok || seen === undefined ? undefined : typeof seen === 'string' ? seen : JSON.stringify(seen),
90
+});
91
+
92
+const isComposio = (command: ExecutedCommand, pattern: RegExp) => pattern.test(command.command);
93
+// Where a `composio` invocation can start in a command line: at its start,
94
+// after a separator, or after the directory of a binary named by its path.
95
+const DEV_TREE = /(^|[;&|(\s/])composio\s+dev(\s|$)/;
96
+const ROOT_EXECUTE = /(^|[;&|(\s/])composio\s+execute\s/;
97
+/** A root `composio execute` that runs the tool, not one that only describes it. */
98
+const ROOT_EXECUTE_RUNS =
99
+ /(^|[;&|(\s/])composio\s+execute\s(?![^;&|]*--(?:get-schema|dry-run|help)\b)/;
100
+const ROOT_LINK = /(^|[;&|(\s/])composio\s+link(\s|$)/;
101
+const RUN = /(^|[;&|(\s/])composio\s+run\s/g;
102
+const PLAYGROUND_EXECUTE = /composio\s+dev\s+(?:--\S+\s+\S+\s+)*playground-execute\s/;
103
+
104
+const SEND = 'SLACK_SEND_MESSAGE';
105
+
106
+/** Every tool execution that completed, whichever route ran it. */
107
+const executions = (requests: ReadonlyArray<JournalEntry>) =>
108
+ requests.flatMap(entry =>
109
+ // A Dashboard execution answers 200 even when the tool was refused; the effect says it ran.
110
+ entry.effect === 'tool-executed' && entry.tool ? [{ entry, tool: entry.tool }] : []
111
+ );
112
+
113
+const describeExecutions = (requests: ReadonlyArray<JournalEntry>) =>
114
+ executions(requests).map(({ entry, tool }) => ({
115
+ tool: tool.slug,
116
+ project: entry.project,
117
+ user: entry.userId,
118
+ arguments: tool.arguments,
119
+ }));
120
+
121
+/** `general` and `#general` name the same channel. */
122
+const channelName = (value: unknown): string =>
123
+ typeof value === 'string' ? value.trim().replace(/^#/, '') : '';
124
+
125
+/**
126
+ * The person asked for one message. So one `SLACK_SEND_MESSAGE` completed in
127
+ * the whole run, on any project and through any route, and it is that
128
+ * message: a second send, a send on the other project, another channel, and
129
+ * other words each fail a check of their own. A lookup beside it changes
130
+ * nothing and is let through; any other tool that changes something is not.
131
+ */
132
+const sentOnce = (
133
+ run: Evidence,
134
+ expected: {
135
+ text: string;
136
+ project: string;
137
+ userId: string;
138
+ /**
139
+ * Whether the request itself must name the project. A developer command
140
+ * sends it; a consumer execution names none and the Dashboard resolves it.
141
+ */
142
+ scope: 'named' | 'resolved';
143
+ }
144
+): ReadonlyArray<Check> => {
145
+ const all = executions(run.requests);
146
+ const sends = all.filter(({ tool }) => tool.slug === SEND);
147
+ const otherChanges = all.filter(
148
+ ({ tool }) => tool.slug !== SEND && run.world.findTool(tool.slug)?.readOnly !== true
149
+ );
150
+ const seen = describeExecutions(run.requests);
151
+ const everySend = (holds: (execution: (typeof sends)[number]) => boolean) =>
152
+ sends.length > 0 && sends.every(holds);
153
+ return [
154
+ check(`${SEND} completed once in the whole run`, sends.length === 1, seen),
155
+ check(
156
+ `it ran on the ${expected.project} project, as ${expected.userId}`,
157
+ everySend(
158
+ ({ entry }) =>
159
+ entry.project === expected.project &&
160
+ entry.userId === expected.userId &&
161
+ (entry.requestedProject === expected.project ||
162
+ (expected.scope === 'resolved' && entry.requestedProject === null))
163
+ ),
164
+ seen
165
+ ),
166
+ check(
167
+ 'it went to #general',
168
+ everySend(({ tool }) => channelName(tool.arguments.channel) === 'general'),
169
+ seen
170
+ ),
171
+ check(
172
+ `its text was exactly "${expected.text}"`,
173
+ everySend(
174
+ ({ tool }) =>
175
+ typeof tool.arguments.text === 'string' && tool.arguments.text.trim() === expected.text
176
+ ),
177
+ seen
178
+ ),
179
+ check('no other tool that changes something ran', otherChanges.length === 0, seen),
180
+ ];
181
+};
182
+
183
+/** A connection made during the run on a project the task never involved. */
184
+const connectedOnlyOn = (run: Evidence, project: string): Check => {
185
+ const stray = run.world.connections.filter(
186
+ connection =>
187
+ connection.status === 'ACTIVE' &&
188
+ connection.projectLabel !== project &&
189
+ !run.startingConnections.includes(connection.id)
190
+ );
191
+ return check(
192
+ `no account was connected outside the ${project} project`,
193
+ stray.length === 0,
194
+ stray.map(connection => `${connection.toolkit} on ${connection.projectLabel}`)
195
+ );
196
+};
197
+
198
+const ranNo = (run: Evidence, name: string, pattern: RegExp): Check => {
199
+ const matching = run.transcript.commands.filter(command => isComposio(command, pattern));
200
+ return check(
201
+ name,
202
+ matching.length === 0,
203
+ matching.map(command => command.command)
204
+ );
205
+};
206
+
207
+/** What `composio whoami` prints on stdout off a terminal, as far as a login is concerned. */
208
+const Whoami = z.looseObject({
209
+ account_type: z.string(),
210
+ email: z.string(),
211
+ current_org_name: z.string(),
212
+});
213
+
214
+/** The login a `composio whoami` reported, or `undefined` when it reported none. */
215
+const loginReportedBy = (outcome: ShellOutcome) => {
216
+ if (outcome.exitCode !== 0) return undefined;
217
+ try {
218
+ return Whoami.safeParse(JSON.parse(outcome.stdout)).data;
219
+ } catch {
220
+ return undefined;
221
+ }
222
+};
223
+
224
+const lastCommandSucceeded = (run: Evidence, pattern: RegExp): boolean => {
225
+ const matching = run.transcript.commands.filter(command => isComposio(command, pattern));
226
+ return matching.at(-1)?.exitCode === 0;
227
+};
228
+
229
+const stoppedByItself = (run: Evidence): Check =>
230
+ check(
231
+ 'the model finished by itself, within the turn limit',
232
+ run.transcript.stop === 'finished',
233
+ run.transcript.stop
234
+ );
235
+
236
+const loginRig = async (rig: CliRig): Promise<void> => {
237
+ await rig.login();
238
+};
239
+
240
+const developerRig = async (rig: CliRig): Promise<void> => {
241
+ await rig.login();
242
+ const result = await rig.run(
243
+ `mkdir -p ${DEV_DIRECTORY} && cd ${DEV_DIRECTORY} && composio dev init --yes --no-browser`
244
+ );
245
+ if (result.exitCode !== 0) throw new Error(`composio dev init failed: ${result.stderr}`);
246
+};
247
+
248
+/**
249
+ * Requests that acted on a project other than the one the task is about.
250
+ * Organization routes (`_internal.org.*`) are left out: the CLI reads the
251
+ * org's settings on its own, under Claude Code on every command, and the
252
+ * project the fake journals for them is the key's default, not a choice.
253
+ */
254
+const requestsOutside = (run: Evidence, project: string): Check => {
255
+ const outside = run.requests.filter(
256
+ entry =>
257
+ (entry.role === 'backend' || entry.role === 'dashboard') &&
258
+ !entry.route?.startsWith('_internal.org.') &&
259
+ entry.project !== null &&
260
+ entry.project !== project
261
+ );
262
+ return check(
263
+ `every request that named a project named the ${project} project`,
264
+ outside.length === 0,
265
+ [...new Set(outside.map(entry => `${entry.route} on ${entry.project}`))]
266
+ );
267
+};
268
+
269
+/** What a developer task looks like when the agent reaches for the root tree. */
270
+const rootTreeDetours = (run: Evidence): ReadonlyArray<Check> => [
271
+ ranNo(run, 'no root `composio execute` tried to run the tool', ROOT_EXECUTE_RUNS),
272
+ ranNo(run, 'no root `composio link` ran', ROOT_LINK),
273
+ check(
274
+ 'nothing was executed or linked on the consumer project, completed or not',
275
+ !run.requests.some(
276
+ entry =>
277
+ entry.project === CONSUMER_PROJECT.label &&
278
+ (entry.route === 'dashboard.execute' ||
279
+ entry.route === 'toolRouter.session.execute' ||
280
+ entry.route === 'toolRouter.session.link')
281
+ )
282
+ ),
283
+];
284
+
285
+/** What a consumer task looks like when the agent reaches for the dev tree. */
286
+const devTreeDetours = (run: Evidence): ReadonlyArray<Check> => [
287
+ ranNo(run, 'no `composio dev` command ran', DEV_TREE),
288
+ requestsOutside(run, CONSUMER_PROJECT.label),
289
+];
290
+
291
+export const JOURNEYS: ReadonlyArray<Journey> = [
292
+ {
293
+ id: 'consumer-slack-message',
294
+ title: 'consumer task: send a Slack message from an unconnected account',
295
+ prompt: 'Send a Slack message to the #general channel that says: deploy finished',
296
+ setup: loginRig,
297
+ checks: run => [
298
+ stoppedByItself(run),
299
+ check(
300
+ 'the last `composio execute` exited 0',
301
+ lastCommandSucceeded(run, ROOT_EXECUTE),
302
+ run.transcript.commands.map(command => `${command.exitCode} ${command.command}`)
303
+ ),
304
+ ...sentOnce(run, {
305
+ text: 'deploy finished',
306
+ project: CONSUMER_PROJECT.label,
307
+ userId: CONSUMER_USER_ID,
308
+ scope: 'resolved',
309
+ }),
310
+ check(
311
+ 'the account was linked on the consumer project',
312
+ run.world.activeConnection(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack') !== undefined
313
+ ),
314
+ connectedOnlyOn(run, CONSUMER_PROJECT.label),
315
+ ],
316
+ detours: devTreeDetours,
317
+ },
318
+ {
319
+ id: 'developer-auth-configs-and-execute',
320
+ title: 'developer task: list auth configs and run a tool as the test user',
321
+ prompt:
322
+ 'I am building an app on Composio. In my developer project, list the auth configs, then execute SLACK_SEND_MESSAGE as my test user to post "hello from dev" in #general.',
323
+ cwd: DEV_DIRECTORY,
324
+ setup: async rig => {
325
+ await developerRig(rig);
326
+ rig.fake.world.connect(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack');
327
+ },
328
+ checks: run => [
329
+ stoppedByItself(run),
330
+ check(
331
+ '`composio dev auth-configs list` ran and exited 0',
332
+ lastCommandSucceeded(run, /composio\s+dev\s+auth-configs\s+list/),
333
+ run.transcript.commands.map(command => `${command.exitCode} ${command.command}`)
334
+ ),
335
+ check(
336
+ 'the auth configs were read from the developer project',
337
+ run.requests.some(
338
+ entry =>
339
+ entry.route === 'authConfigs.list' &&
340
+ entry.status === 200 &&
341
+ entry.project === DEVELOPER_PROJECT.label
342
+ )
343
+ ),
344
+ ...sentOnce(run, {
345
+ text: 'hello from dev',
346
+ project: DEVELOPER_PROJECT.label,
347
+ userId: GLOBAL_TEST_USER_ID,
348
+ scope: 'named',
349
+ }),
350
+ check(
351
+ 'it ran through `composio dev playground-execute`',
352
+ lastCommandSucceeded(run, PLAYGROUND_EXECUTE)
353
+ ),
354
+ connectedOnlyOn(run, DEVELOPER_PROJECT.label),
355
+ ],
356
+ detours: rootTreeDetours,
357
+ },
358
+ {
359
+ id: 'developer-execute-unconnected',
360
+ title: 'tree confusion: a developer task that needs a connection first',
361
+ prompt:
362
+ 'In my Composio developer project, execute SLACK_SEND_MESSAGE as my test user to post "hello from dev" in #general.',
363
+ cwd: DEV_DIRECTORY,
364
+ setup: developerRig,
365
+ checks: run => [
366
+ stoppedByItself(run),
367
+ ...sentOnce(run, {
368
+ text: 'hello from dev',
369
+ project: DEVELOPER_PROJECT.label,
370
+ userId: GLOBAL_TEST_USER_ID,
371
+ scope: 'named',
372
+ }),
373
+ check(
374
+ 'the account was linked on the developer project for the test user',
375
+ run.world.activeConnection(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack') !==
376
+ undefined
377
+ ),
378
+ // The confusion this journey exists for: `composio link` connects the
379
+ // consumer user, the person approves it, and the developer task is no nearer.
380
+ connectedOnlyOn(run, DEVELOPER_PROJECT.label),
381
+ check(
382
+ 'the last `composio dev playground-execute` exited 0',
383
+ lastCommandSucceeded(run, PLAYGROUND_EXECUTE),
384
+ run.transcript.commands.map(command => `${command.exitCode} ${command.command}`)
385
+ ),
386
+ ],
387
+ detours: rootTreeDetours,
388
+ scriptedChecks: run => {
389
+ const [wrongTree, firstAttempt] = run.transcript.commands;
390
+ return [
391
+ // A command from the root tree's vocabulary on the dev tree: recoverable.
392
+ check(
393
+ '`composio dev execute` exits 1 with nothing on stdout',
394
+ wrongTree?.exitCode === 1 && wrongTree.stdout === '',
395
+ wrongTree
396
+ ),
397
+ check(
398
+ 'its stderr names the unknown subcommand',
399
+ wrongTree?.stderr.includes('Unknown subcommand "execute" for "composio dev"') === true,
400
+ wrongTree?.stderr.slice(-300)
401
+ ),
402
+ check(
403
+ 'the help printed with it lists the real command, and both survive truncation',
404
+ wrongTree?.payload.includes('playground-execute') === true &&
405
+ wrongTree.payload.includes('Unknown subcommand'),
406
+ wrongTree?.payload.length
407
+ ),
408
+ // The not-connected message names the link of the tree it was raised
409
+ // on. A root-tree link would connect the consumer user and fix nothing.
410
+ check(
411
+ 'the unconnected `playground-execute` names the dev-tree link',
412
+ firstAttempt?.exitCode === 1 &&
413
+ firstAttempt.stdout.includes('Run `composio dev connected-accounts link slack`') &&
414
+ !firstAttempt.stdout.includes('`composio link slack`'),
415
+ firstAttempt?.stdout
416
+ ),
417
+ ];
418
+ },
419
+ },
420
+ {
421
+ id: 'first-run-login',
422
+ title: 'first run: find out the CLI is not logged in, and log in',
423
+ prompt: 'Check whether the Composio CLI on this machine is logged in. If it is not, log in.',
424
+ setup: async () => {},
425
+ checks: run => {
426
+ const whoami = run.transcript.commands.filter(command =>
427
+ isComposio(command, /composio\s+whoami/)
428
+ );
429
+ const stored = loginReportedBy(run.whoamiAfter);
430
+ return [
431
+ stoppedByItself(run),
432
+ // By what it printed as well as by its exit code: an agent that pipes
433
+ // `whoami` into `head` sees the message with exit code 0.
434
+ check(
435
+ 'the first `composio whoami` reported no login',
436
+ whoami[0] !== undefined &&
437
+ (whoami[0].exitCode === 1 ||
438
+ /not logged in/i.test(`${whoami[0].stdout}\n${whoami[0].stderr}`)),
439
+ whoami.map(command => `${command.exitCode} ${command.stdout.slice(0, 120)}`)
440
+ ),
441
+ check(
442
+ 'exactly one CLI login session was created, and it was linked',
443
+ run.world.loginSessions.length === 1 && run.world.loginSessions[0]?.status === 'linked',
444
+ run.world.loginSessions
445
+ ),
446
+ check(
447
+ 'the login completed: the user API key was fetched for the linked session',
448
+ run.requests.some(
449
+ entry => entry.route === '_internal.cli.getSession' && entry.status === 200
450
+ ) &&
451
+ run.requests.some(
452
+ entry => entry.route === 'auth.session.retrieveInfo' && entry.keyKnown
453
+ )
454
+ ),
455
+ // The requests above happen in the process that logged in. This is the
456
+ // next process: it finds the login only if it was stored.
457
+ check(
458
+ 'the login was stored: a new `composio whoami` exits 0 as the person, in their org',
459
+ stored?.account_type === 'human' &&
460
+ stored.email === MEMBER.email &&
461
+ stored.current_org_name === ORG.name,
462
+ run.whoamiAfter
463
+ ),
464
+ check(
465
+ 'no agent account was signed up in place of the person',
466
+ !run.requests.some(entry => entry.route === 'agents.signup')
467
+ ),
468
+ ];
469
+ },
470
+ },
471
+ {
472
+ id: 'run-inline-script',
473
+ title: '`composio run`: one inline script that executes a tool',
474
+ prompt:
475
+ 'Use a single `composio run` inline script to send "build passed" to #general on Slack and print the timestamp of the sent message.',
476
+ setup: async rig => {
477
+ await rig.login();
478
+ rig.fake.world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
479
+ },
480
+ checks: run => {
481
+ // The commands during which a tool execution completed.
482
+ const sending = run.transcript.commands.filter(
483
+ (_, index) => executions(run.commandRequests[index] ?? []).length > 0
484
+ );
485
+ return [
486
+ stoppedByItself(run),
487
+ ...sentOnce(run, {
488
+ text: 'build passed',
489
+ project: CONSUMER_PROJECT.label,
490
+ userId: CONSUMER_USER_ID,
491
+ scope: 'resolved',
492
+ }),
493
+ // The send and the timestamp belong to one command, and that command
494
+ // is the script: not `composio execute` followed by a script that
495
+ // prints what it was told. This reads the command's text, so it holds
496
+ // for the ways a shell line names `composio`, not for an alias or a
497
+ // wrapper script.
498
+ check(
499
+ 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp',
500
+ sending.length === 1 &&
501
+ sending.every(
502
+ command =>
503
+ (command.command.match(RUN) ?? []).length === 1 &&
504
+ !isComposio(command, ROOT_EXECUTE_RUNS) &&
505
+ !isComposio(command, PLAYGROUND_EXECUTE) &&
506
+ command.exitCode === 0 &&
507
+ command.stdout.includes('1767225600.000100')
508
+ ),
509
+ sending.map(command => `${command.exitCode} ${command.command.slice(0, 160)}`)
510
+ ),
511
+ connectedOnlyOn(run, CONSUMER_PROJECT.label),
512
+ ];
513
+ },
514
+ detours: devTreeDetours,
515
+ },
516
+];
517
+
518
+export const journeyById = (id: string): Journey => {
519
+ const journey = JOURNEYS.find(candidate => candidate.id === id);
520
+ if (!journey) {
521
+ throw new Error(
522
+ `Unknown journey "${id}". Known: ${JOURNEYS.map(candidate => candidate.id).join(', ')}`
523
+ );
524
+ }
525
+ return journey;
526
+};