@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 403 lines 14.3 KB
Raw
1 /**
2 * The graders, without a container: a run that reached the goal passes, and
3 * each way of looking as if it did fails the check that names it.
4 */
5
6 import { describe, expect, it } from 'bun:test';
7 import {
8 CONSUMER_PROJECT,
9 CONSUMER_USER_ID,
10 DEVELOPER_PROJECT,
11 GLOBAL_TEST_USER_ID,
12 MEMBER,
13 ORG,
14 World,
15 type JournalEntry,
16 } from '@e2e-tests/cli-fake-backend';
17 import { toolPayload, type ExecutedCommand, type ShellOutcome } from './src/agent';
18 import { journeyById, JOURNEYS, type Evidence } from './src/journeys';
19
20 const TIMESTAMP = '1767225600.000100';
21 const LOGGED_IN: ShellOutcome = {
22 exitCode: 0,
23 stdout: `{"account_type":"human","email":"${MEMBER.email}","current_org_name":"${ORG.name}"}\n`,
24 stderr: '',
25 };
26
27 let seq = 0;
28 const request = (fields: Partial<JournalEntry>): JournalEntry => ({
29 seq: (seq += 1),
30 role: 'backend',
31 method: 'POST',
32 path: '/',
33 query: {},
34 route: null,
35 keyClass: 'user',
36 keyKnown: true,
37 orgId: ORG.id,
38 project: null,
39 requestedProject: null,
40 projectFrom: null,
41 userId: null,
42 body: null,
43 status: 200,
44 effect: null,
45 tool: null,
46 fault: null,
47 ...fields,
48 });
49
50 const send = (
51 tree: 'consumer' | 'developer',
52 args: Record<string, unknown>,
53 fields: Partial<JournalEntry> = {}
54 ): JournalEntry =>
55 request({
56 role: tree === 'consumer' ? 'dashboard' : 'backend',
57 route: tree === 'consumer' ? 'dashboard.execute' : 'toolRouter.session.execute',
58 project: tree === 'consumer' ? CONSUMER_PROJECT.label : DEVELOPER_PROJECT.label,
59 // A developer command names its project; the Dashboard resolves the consumer one.
60 requestedProject: tree === 'consumer' ? null : DEVELOPER_PROJECT.label,
61 userId: tree === 'consumer' ? CONSUMER_USER_ID : GLOBAL_TEST_USER_ID,
62 effect: 'tool-executed',
63 tool: { slug: 'SLACK_SEND_MESSAGE', arguments: args },
64 ...fields,
65 });
66
67 /** One shell command and the requests it caused. */
68 const step = (
69 command: string,
70 requests: ReadonlyArray<JournalEntry> = [],
71 outcome: Partial<ShellOutcome> = {}
72 ) => {
73 const result: ShellOutcome = { exitCode: 0, stdout: '', stderr: '', ...outcome };
74 const executed: ExecutedCommand = { command, ...result, payload: toolPayload(result) };
75 return { executed, requests };
76 };
77
78 const evidence = (
79 world: World,
80 steps: ReadonlyArray<ReturnType<typeof step>>,
81 overrides: Partial<Evidence> = {}
82 ): Evidence => ({
83 transcript: {
84 commands: steps.map(taken => taken.executed),
85 messages: [],
86 replies: [],
87 stop: 'finished',
88 modelTurns: steps.length + 1,
89 },
90 world,
91 requests: steps.flatMap(taken => taken.requests),
92 commandRequests: steps.map(taken => taken.requests),
93 startingConnections: [],
94 whoamiAfter: LOGGED_IN,
95 ...overrides,
96 });
97
98 const failed = (id: string, run: Evidence): string[] =>
99 journeyById(id)
100 .checks(run)
101 .filter(check => !check.ok)
102 .map(check => check.name);
103
104 const detoursTaken = (id: string, run: Evidence): string[] =>
105 (journeyById(id).detours?.(run) ?? []).filter(check => !check.ok).map(check => check.name);
106
107 const DEV_SEND = `composio dev playground-execute SLACK_SEND_MESSAGE -d '{}'`;
108 const ROOT_SEND = `composio execute SLACK_SEND_MESSAGE -d '{}'`;
109 const hello = { channel: 'general', text: 'hello from dev' };
110
111 describe('developer-execute-unconnected', () => {
112 const id = 'developer-execute-unconnected';
113 const connected = () => {
114 const world = new World();
115 world.connect(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack');
116 return world;
117 };
118
119 it('passes when the message was sent once, on the developer project', () => {
120 expect(failed(id, evidence(connected(), [step(DEV_SEND, [send('developer', hello)])]))).toEqual(
121 []
122 );
123 });
124
125 it('fails when the message was also sent on the consumer project', () => {
126 const world = connected();
127 world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
128 const run = evidence(world, [
129 step('composio link slack'),
130 step(ROOT_SEND, [send('consumer', hello)]),
131 step(DEV_SEND, [send('developer', hello)]),
132 ]);
133 expect(failed(id, run)).toEqual([
134 'SLACK_SEND_MESSAGE completed once in the whole run',
135 `it ran on the ${DEVELOPER_PROJECT.label} project, as ${GLOBAL_TEST_USER_ID}`,
136 `no account was connected outside the ${DEVELOPER_PROJECT.label} project`,
137 ]);
138 });
139
140 it('fails when the person approved a consumer connection, even with nothing sent there', () => {
141 const world = connected();
142 world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
143 const run = evidence(world, [
144 step('composio link slack'),
145 step(DEV_SEND, [send('developer', hello)]),
146 ]);
147 expect(failed(id, run)).toEqual([
148 `no account was connected outside the ${DEVELOPER_PROJECT.label} project`,
149 ]);
150 });
151
152 it('passes after a wrong-tree attempt that changed nothing, and reports the detour', () => {
153 const attempt = request({
154 role: 'dashboard',
155 route: 'dashboard.execute',
156 project: CONSUMER_PROJECT.label,
157 userId: CONSUMER_USER_ID,
158 });
159 const run = evidence(connected(), [
160 step(ROOT_SEND, [attempt], { exitCode: 1 }),
161 step(DEV_SEND, [send('developer', hello)]),
162 ]);
163 expect(failed(id, run)).toEqual([]);
164 expect(detoursTaken(id, run)).toEqual([
165 'no root `composio execute` tried to run the tool',
166 'nothing was executed or linked on the consumer project, completed or not',
167 ]);
168 });
169
170 it('does not count describing the tool on the root tree as trying to run it', () => {
171 const run = evidence(connected(), [
172 step('composio execute SLACK_SEND_MESSAGE --get-schema'),
173 step(DEV_SEND, [send('developer', hello)]),
174 ]);
175 expect(detoursTaken(id, run)).toEqual([]);
176 });
177
178 it('fails when the request named another project, or none, and the session decided', () => {
179 const scopeCheck = `it ran on the ${DEVELOPER_PROJECT.label} project, as ${GLOBAL_TEST_USER_ID}`;
180 const crossed = send('developer', hello, { requestedProject: CONSUMER_PROJECT.label });
181 const unscoped = send('developer', hello, { requestedProject: null });
182 expect(failed(id, evidence(connected(), [step(DEV_SEND, [crossed])]))).toEqual([scopeCheck]);
183 expect(failed(id, evidence(connected(), [step(DEV_SEND, [unscoped])]))).toEqual([scopeCheck]);
184 });
185 });
186
187 describe('consumer-slack-message', () => {
188 const id = 'consumer-slack-message';
189 const connected = () => {
190 const world = new World();
191 world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
192 return world;
193 };
194 const sent = (args: Record<string, unknown>) =>
195 evidence(connected(), [step(ROOT_SEND, [send('consumer', args)])]);
196
197 it('passes for #general or general, with the text as asked', () => {
198 expect(failed(id, sent({ channel: '#general', text: 'deploy finished' }))).toEqual([]);
199 expect(failed(id, sent({ channel: 'general', text: 'deploy finished' }))).toEqual([]);
200 });
201
202 it('fails on another channel', () => {
203 expect(failed(id, sent({ channel: 'private-leadership', text: 'deploy finished' }))).toEqual([
204 'it went to #general',
205 ]);
206 });
207
208 it('fails on other words around the text', () => {
209 expect(failed(id, sent({ channel: '#general', text: 'deploy finished WRONG TEXT' }))).toEqual([
210 'its text was exactly "deploy finished"',
211 ]);
212 });
213
214 it('fails on a second send, whatever it said', () => {
215 const run = evidence(connected(), [
216 step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]),
217 step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]),
218 ]);
219 expect(failed(id, run)).toEqual(['SLACK_SEND_MESSAGE completed once in the whole run']);
220 });
221
222 it('sees a send through the direct execute route', () => {
223 const direct = send(
224 'consumer',
225 { channel: '#general', text: 'deploy finished' },
226 { role: 'backend', route: 'tools.execute' }
227 );
228 const run = evidence(connected(), [
229 step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]),
230 step('curl …', [direct]),
231 ]);
232 expect(failed(id, run)).toContain('SLACK_SEND_MESSAGE completed once in the whole run');
233 });
234
235 it('lets a lookup through, and fails on another tool that changes something', () => {
236 const other = (slug: string) => send('consumer', {}, { tool: { slug, arguments: {} } });
237 const withTool = (slug: string) =>
238 evidence(connected(), [
239 step('composio execute …', [other(slug)]),
240 step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]),
241 ]);
242 expect(failed(id, withTool('HACKERNEWS_GET_USER'))).toEqual([]);
243 expect(failed(id, withTool('GMAIL_SEND_EMAIL'))).toEqual([
244 'no other tool that changes something ran',
245 ]);
246 });
247
248 it('reports a look at the dev tree as a detour, not a failure', () => {
249 const run = evidence(connected(), [
250 step('composio dev --help'),
251 step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]),
252 ]);
253 expect(failed(id, run)).toEqual([]);
254 expect(detoursTaken(id, run)).toEqual(['no `composio dev` command ran']);
255 });
256 });
257
258 describe('run-inline-script', () => {
259 const id = 'run-inline-script';
260 const built = { channel: '#general', text: 'build passed' };
261 const world = () => {
262 const seeded = new World();
263 const connection = seeded.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
264 return { seeded, startingConnections: [connection.id] };
265 };
266 const SCRIPT = `composio run 'const r = await execute("SLACK_SEND_MESSAGE", {}); console.log(r.data.ts)'`;
267
268 it('passes when the script sent the message and printed its timestamp', () => {
269 const { seeded, startingConnections } = world();
270 const run = evidence(
271 seeded,
272 [step(SCRIPT, [send('consumer', built)], { stdout: `${TIMESTAMP}\n` })],
273 { startingConnections }
274 );
275 expect(failed(id, run)).toEqual([]);
276 });
277
278 it("takes no detour for the CLI's own read of the org's settings", () => {
279 const { seeded, startingConnections } = world();
280 const orgRead = request({
281 route: '_internal.org.consumer.retrieveConfig',
282 project: DEVELOPER_PROJECT.label,
283 method: 'GET',
284 });
285 const run = evidence(
286 seeded,
287 [step(SCRIPT, [orgRead, send('consumer', built)], { stdout: `${TIMESTAMP}\n` })],
288 { startingConnections }
289 );
290 expect(detoursTaken(id, run)).toEqual([]);
291 });
292
293 it('fails when `composio execute` sent it and a script only printed the timestamp', () => {
294 const { seeded, startingConnections } = world();
295 const run = evidence(
296 seeded,
297 [
298 step(`composio execute SLACK_SEND_MESSAGE -d '{}'`, [send('consumer', built)]),
299 step(`composio run 'console.log("${TIMESTAMP}")'`, [], { stdout: `${TIMESTAMP}\n` }),
300 ],
301 { startingConnections }
302 );
303 expect(failed(id, run)).toEqual([
304 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp',
305 ]);
306 });
307
308 it('fails when the binary is named by its path, or the line runs two scripts', () => {
309 const { seeded, startingConnections } = world();
310 const one = (command: string) =>
311 evidence(seeded, [step(command, [send('consumer', built)], { stdout: `${TIMESTAMP}\n` })], {
312 startingConnections,
313 });
314 const failure =
315 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp';
316 expect(
317 failed(
318 id,
319 one(
320 `/usr/local/bin/composio execute SLACK_SEND_MESSAGE -d '{}'; composio run 'console.log("${TIMESTAMP}")'`
321 )
322 )
323 ).toEqual([failure]);
324 expect(failed(id, one(`${SCRIPT}; composio run 'console.log(1)'`))).toEqual([failure]);
325 });
326
327 it('fails when both happened in one shell command', () => {
328 const { seeded, startingConnections } = world();
329 const run = evidence(
330 seeded,
331 [
332 step(
333 `composio execute SLACK_SEND_MESSAGE -d '{}'; composio run 'console.log("${TIMESTAMP}")'`,
334 [send('consumer', built)],
335 { stdout: `${TIMESTAMP}\n` }
336 ),
337 ],
338 { startingConnections }
339 );
340 expect(failed(id, run)).toEqual([
341 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp',
342 ]);
343 });
344 });
345
346 describe('first-run-login', () => {
347 const id = 'first-run-login';
348 const loggedIn = () => {
349 const world = new World();
350 world.createLoginSession();
351 world.linkLoginSession();
352 return [
353 world,
354 [
355 step('composio whoami', [], { exitCode: 1 }),
356 step('composio login --poll', [
357 request({ method: 'GET', route: '_internal.cli.getSession' }),
358 request({ method: 'GET', route: 'auth.session.retrieveInfo' }),
359 ]),
360 ],
361 ] as const;
362 };
363
364 it('passes when the next process finds the login', () => {
365 const [world, steps] = loggedIn();
366 expect(failed(id, evidence(world, steps))).toEqual([]);
367 });
368
369 it('reads the login from the JSON, however it is laid out', () => {
370 const [world, steps] = loggedIn();
371 const pretty = JSON.stringify(JSON.parse(LOGGED_IN.stdout), null, 2);
372 expect(
373 failed(id, evidence(world, steps, { whoamiAfter: { ...LOGGED_IN, stdout: pretty } }))
374 ).toEqual([]);
375 });
376
377 it('fails when the key was fetched and nothing was stored', () => {
378 const [world, steps] = loggedIn();
379 const run = evidence(world, steps, {
380 whoamiAfter: { exitCode: 1, stdout: '', stderr: 'You are not logged in yet.' },
381 });
382 expect(failed(id, run)).toEqual([
383 'the login was stored: a new `composio whoami` exits 0 as the person, in their org',
384 ]);
385 });
386 });
387
388 // Harbor's "nop" agent: a grader that an agent passes by doing nothing measures nothing.
389 describe('an agent that does nothing', () => {
390 for (const journey of JOURNEYS) {
391 it(`fails ${journey.id}`, () => {
392 const world = new World();
393 // The account the journey's setup may have connected is already there.
394 world.connect(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack');
395 world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
396 const idle = evidence(world, [], {
397 startingConnections: world.connections.map(connection => connection.id),
398 whoamiAfter: { exitCode: 1, stdout: '', stderr: 'You are not logged in yet.' },
399 });
400 expect(failed(journey.id, idle)).not.toEqual([]);
401 });
402 }
403 });