@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 292 lines 9.6 KB
Raw
1 /**
2 * The loop agents live in, without a model: search, execute, link, execute
3 * again, then the commands around it (`connections`, `--get-schema`,
4 * `--dry-run`, `run`). Each step checks what the fake backend received, so a
5 * step that printed success without acting fails here.
6 */
7
8 import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
9 import { CONSUMER_PROJECT, CONSUMER_USER_ID } from '@e2e-tests/cli-fake-backend';
10 import { startCliRig, type CliRig } from '@e2e-tests/cli-fake-backend/rig';
11 import { z } from 'zod';
12 import {
13 clean,
14 commandRequests,
15 EXECUTE_SLACK,
16 ExecutionResult,
17 RIG_TIMEOUT_MS,
18 routesOf,
19 SEARCH_SLACK,
20 SearchResult,
21 stdoutJson,
22 TEST_TIMEOUT_MS,
23 writeDebugLog,
24 } from './support';
25
26 const Connections = z.record(
27 z.string(),
28 z.array(z.looseObject({ id: z.string(), status: z.string() }))
29 );
30
31 describe('tool loop: search, execute, link, execute', () => {
32 let rig: CliRig;
33
34 beforeAll(async () => {
35 rig = await startCliRig({ label: 'tool-loop' });
36 await rig.login();
37 }, RIG_TIMEOUT_MS);
38
39 afterAll(async () => {
40 await writeDebugLog(rig, import.meta.dir, 'tool-loop');
41 await rig.stop();
42 });
43
44 it(
45 '`search` finds the slug and names the next commands',
46 async () => {
47 const result = await rig.run(SEARCH_SLACK);
48 expect(result.exitCode).toBe(0);
49 expect(result.stderr).toBe('');
50 const found = stdoutJson(result, SearchResult);
51 expect(found.results[0]?.primary_tool_slugs).toEqual(['SLACK_SEND_MESSAGE']);
52 // Not connected yet, so the first suggested step is the link.
53 expect(found.next_steps.steps.map(step => step.command)[0]).toBe('composio link slack');
54
55 const search = result.requests.find(entry => entry.route === 'toolRouter.session.search');
56 expect(search).toMatchObject({ project: CONSUMER_PROJECT.label, userId: CONSUMER_USER_ID });
57 expect(search?.body).toEqual({ queries: [{ use_case: 'send a slack message' }] });
58 },
59 TEST_TIMEOUT_MS
60 );
61
62 it(
63 '`execute` before linking fails with the command that fixes it',
64 async () => {
65 const result = await rig.run(EXECUTE_SLACK);
66 expect(result.exitCode).toBe(1);
67 expect(result.stderr).toBe('');
68 expect(stdoutJson(result, ExecutionResult)).toMatchObject({
69 successful: false,
70 error:
71 'No active connection found for toolkit "slack". Run `composio link slack`, then retry.',
72 slug: 'ToolRouterV2_NoActiveConnection',
73 });
74 },
75 TEST_TIMEOUT_MS
76 );
77
78 it(
79 '`connections list` shows nothing connected yet',
80 async () => {
81 const result = await rig.run('composio connections list');
82 expect(result.exitCode).toBe(0);
83 expect(stdoutJson(result, Connections)).toEqual({});
84 },
85 TEST_TIMEOUT_MS
86 );
87
88 it(
89 '`link` creates the connection for the consumer user and waits until it is active',
90 async () => {
91 const result = await rig.run('composio link slack');
92 expect(result.exitCode).toBe(0);
93 expect(
94 stdoutJson(result, z.looseObject({ status: z.string(), toolkit: z.string() }))
95 ).toMatchObject({ status: 'success', toolkit: 'slack' });
96
97 const link = result.requests.filter(entry => entry.route === 'toolRouter.session.link');
98 expect(link).toHaveLength(1);
99 expect(link[0]).toMatchObject({
100 project: CONSUMER_PROJECT.label,
101 userId: CONSUMER_USER_ID,
102 body: { toolkit: 'slack' },
103 });
104 expect(routesOf(result)).toContain('connectedAccounts.retrieve');
105 expect(
106 rig.fake.world.activeConnection(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack')
107 ).toBeDefined();
108 },
109 TEST_TIMEOUT_MS
110 );
111
112 it(
113 '`execute` after linking runs the tool with the given arguments',
114 async () => {
115 const result = await rig.run(EXECUTE_SLACK);
116 expect(result.exitCode).toBe(0);
117 expect(stdoutJson(result, ExecutionResult)).toMatchObject({
118 successful: true,
119 data: { ok: true, channel: 'general', message: { text: 'hello' } },
120 });
121
122 const execution = result.requests.filter(entry => entry.route === 'dashboard.execute');
123 expect(execution).toHaveLength(1);
124 expect(execution[0]?.body).toMatchObject({
125 json: { tool_slug: 'SLACK_SEND_MESSAGE', arguments: { channel: 'general', text: 'hello' } },
126 });
127 },
128 TEST_TIMEOUT_MS
129 );
130
131 it(
132 '`connections list` shows the linked account as active',
133 async () => {
134 const result = await rig.run('composio connections list');
135 expect(result.exitCode).toBe(0);
136 const connections = stdoutJson(result, Connections);
137 expect(Object.keys(connections)).toEqual(['slack']);
138 expect(connections.slack?.map(account => account.status)).toEqual(['ACTIVE']);
139 },
140 TEST_TIMEOUT_MS
141 );
142 });
143
144 describe('tool loop: looking before acting', () => {
145 let rig: CliRig;
146
147 beforeAll(async () => {
148 rig = await startCliRig({ label: 'tool-inspect' });
149 await rig.login();
150 rig.fake.world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
151 }, RIG_TIMEOUT_MS);
152
153 afterAll(async () => {
154 await writeDebugLog(rig, import.meta.dir, 'tool-inspect');
155 await rig.stop();
156 });
157
158 const executions = (result: Awaited<ReturnType<CliRig['run']>>) =>
159 result.requests.filter(entry => entry.route === 'dashboard.execute');
160
161 it(
162 '`--get-schema` prints the input schema and executes nothing',
163 async () => {
164 const result = await rig.run('composio execute SLACK_SEND_MESSAGE --get-schema');
165 expect(result.exitCode).toBe(0);
166 const schema = stdoutJson(
167 result,
168 z.looseObject({
169 slug: z.string(),
170 inputSchema: z.looseObject({ required: z.array(z.string()) }),
171 })
172 );
173 expect(schema.slug).toBe('SLACK_SEND_MESSAGE');
174 expect(schema.inputSchema.required).toEqual(['channel', 'text']);
175 expect(executions(result)).toEqual([]);
176 },
177 TEST_TIMEOUT_MS
178 );
179
180 it(
181 '`--dry-run` validates and reports the acting user without executing',
182 async () => {
183 const result = await rig.run(`${EXECUTE_SLACK} --dry-run`);
184 expect(result.exitCode).toBe(0);
185 expect(
186 stdoutJson(result, z.looseObject({ dryRun: z.boolean(), userId: z.string() }))
187 ).toMatchObject({ dryRun: true, userId: CONSUMER_USER_ID });
188 expect(executions(result)).toEqual([]);
189 },
190 TEST_TIMEOUT_MS
191 );
192
193 it(
194 'arguments that miss a required field are rejected before the tool runs',
195 async () => {
196 const result = await rig.run(
197 `composio execute SLACK_SEND_MESSAGE -d '{"channel":"general"}'`
198 );
199 expect(result.exitCode).toBe(1);
200 const outcome = stdoutJson(result, ExecutionResult);
201 expect(outcome.successful).toBe(false);
202 expect(outcome.error).toContain('text');
203 expect(executions(result)).toEqual([]);
204 },
205 TEST_TIMEOUT_MS
206 );
207
208 it(
209 'an unknown tool slug fails with the backend message',
210 async () => {
211 const result = await rig.run(`composio execute SLACK_NO_SUCH_TOOL -d '{}'`);
212 expect(result.exitCode).toBe(1);
213 expect(stdoutJson(result, ExecutionResult)).toMatchObject({
214 successful: false,
215 error: 'Tool SLACK_NO_SUCH_TOOL not found',
216 });
217 },
218 TEST_TIMEOUT_MS
219 );
220 });
221
222 describe('tool loop: `run` with an inline script', () => {
223 let rig: CliRig;
224
225 beforeAll(async () => {
226 rig = await startCliRig({ label: 'run' });
227 await rig.login();
228 rig.fake.world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
229 }, RIG_TIMEOUT_MS);
230
231 afterAll(async () => {
232 await writeDebugLog(rig, import.meta.dir, 'run');
233 await rig.stop();
234 });
235
236 it(
237 'the injected `execute()` runs the tool on the consumer project and returns its result',
238 async () => {
239 const result = await rig.run(
240 `composio run 'const sent = await execute("SLACK_SEND_MESSAGE", { channel: "general", text: "from run" }); console.log(JSON.stringify({ ok: sent.data.ok, text: sent.data.message.text }));'`
241 );
242 expect(result.exitCode).toBe(0);
243 expect(result.stdout.trim()).toBe('{"ok":true,"text":"from run"}');
244
245 const execution = result.requests.filter(entry => entry.route === 'dashboard.execute');
246 expect(execution).toHaveLength(1);
247 expect(execution[0]).toMatchObject({
248 project: CONSUMER_PROJECT.label,
249 userId: CONSUMER_USER_ID,
250 keyClass: 'user',
251 });
252 },
253 TEST_TIMEOUT_MS
254 );
255
256 it(
257 'the injected `search()` returns what `composio search` prints',
258 async () => {
259 const result = await rig.run(
260 `composio run 'const found = await search("send a slack message"); console.log(found.results[0].primary_tool_slugs.join(","));'`
261 );
262 expect(result.exitCode).toBe(0);
263 expect(result.stdout.trim()).toBe('SLACK_SEND_MESSAGE');
264 expect(routesOf(result)).toContain('toolRouter.session.search');
265 },
266 TEST_TIMEOUT_MS
267 );
268
269 it(
270 'an `execute()` that is not connected fails the script with the link command on stderr',
271 async () => {
272 const result = await rig.run(
273 `composio run 'await execute("GITHUB_GET_THE_AUTHENTICATED_USER"); console.log("unreachable");'`
274 );
275 expect(result.exitCode).toBe(1);
276 expect(result.stdout).toBe('');
277 expect(clean(result.stderr)).toContain('Run `composio link github`, then retry.');
278 },
279 TEST_TIMEOUT_MS
280 );
281
282 it(
283 'a script that never calls a helper makes no tool request',
284 async () => {
285 const result = await rig.run(`composio run 'console.log(1 + 1)'`);
286 expect(result.exitCode).toBe(0);
287 expect(result.stdout.trim()).toBe('2');
288 expect(commandRequests(result).filter(entry => entry.route?.includes('execute'))).toEqual([]);
289 },
290 TEST_TIMEOUT_MS
291 );
292 });