@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 292 lines 9.16 KB
Raw
1 /**
2 * Network faults: what the CLI does when a host it depends on misbehaves.
3 *
4 * Two invariants carry most of the weight. A read may be retried, and either
5 * heals or fails with exit 1, empty stdout, and the cause on stderr. A tool
6 * execution is sent exactly once whatever happens, because a second attempt
7 * could act twice.
8 *
9 * A failure that is safe to send again as it is (a 429, which the API refused
10 * before acting) exits 75 instead of 1, so a caller can retry on the code alone.
11 * A failed execution says in its result whether it is safe to run again or may
12 * have run (`retry`).
13 */
14
15 import { afterAll, afterEach, beforeAll, describe, expect, it } from 'bun:test';
16 import { CONSUMER_PROJECT, CONSUMER_USER_ID, type Fault } from '@e2e-tests/cli-fake-backend';
17 import { startCliRig, type CliRig } from '@e2e-tests/cli-fake-backend/rig';
18 import {
19 clean,
20 EXECUTE_SLACK,
21 ExecutionResult,
22 RIG_TIMEOUT_MS,
23 SEARCH_SLACK,
24 SearchResult,
25 stdoutJson,
26 TEST_TIMEOUT_MS,
27 writeDebugLog,
28 } from './support';
29
30 const SEARCH = 'toolRouter.session.search';
31
32 /** The exit code of a failure that is safe to send again as it is (`EX_TEMPFAIL`). */
33 const TEMPORARY_FAILURE = 75;
34 const RETRY_HINT = 'run the same command again';
35
36 describe('network faults on a read (`search`)', () => {
37 let rig: CliRig;
38
39 beforeAll(async () => {
40 rig = await startCliRig({ label: 'net-read' });
41 await rig.login();
42 }, RIG_TIMEOUT_MS);
43
44 afterEach(() => rig.fake.faults.clear());
45
46 afterAll(async () => {
47 await writeDebugLog(rig, import.meta.dir, 'net-read');
48 await rig.stop();
49 });
50
51 const searchRequests = (result: Awaited<ReturnType<CliRig['run']>>) =>
52 result.requests.filter(entry => entry.route === SEARCH);
53
54 it(
55 'connection refused: fails with the cause and an empty stdout',
56 async () => {
57 // Port 9 (discard) has no listener on the host.
58 const result = await rig.run(
59 `COMPOSIO_BASE_URL=http://host.docker.internal:9 ${SEARCH_SLACK}`
60 );
61 expect(result.exitCode).toBe(1);
62 expect(result.stdout).toBe('');
63 expect(clean(result.stderr)).toContain('ECONNREFUSED');
64 expect(result.requests).toEqual([]);
65 },
66 TEST_TIMEOUT_MS
67 );
68
69 const persistent: ReadonlyArray<{
70 name: string;
71 fault: Fault;
72 stderr: ReadonlyArray<string>;
73 attempts: number;
74 exitCode?: number;
75 }> = [
76 {
77 name: 'HTTP 500 on every attempt',
78 fault: { kind: 'status', status: 500 },
79 stderr: ['Injected HTTP 500'],
80 attempts: 3,
81 },
82 {
83 name: 'HTTP 429 on every attempt',
84 fault: { kind: 'status', status: 429, headers: { 'retry-after': '1' } },
85 stderr: ['Injected HTTP 429', RETRY_HINT],
86 attempts: 3,
87 exitCode: TEMPORARY_FAILURE,
88 },
89 {
90 name: 'an HTML error page from a proxy',
91 fault: { kind: 'html', status: 502 },
92 stderr: ['502'],
93 attempts: 3,
94 },
95 {
96 name: 'connection reset before any response',
97 fault: { kind: 'reset' },
98 stderr: ['ECONNRESET'],
99 attempts: 3,
100 },
101 {
102 name: 'connection reset in the middle of the body',
103 fault: { kind: 'reset-mid-body' },
104 stderr: ['ECONNRESET'],
105 attempts: 1,
106 },
107 {
108 name: 'a body that is not JSON',
109 fault: { kind: 'malformed-json' },
110 stderr: ['is not in the shape this version of the CLI expects', 'composio upgrade'],
111 attempts: 1,
112 },
113 ];
114
115 for (const { name, fault, stderr, attempts, exitCode = 1 } of persistent) {
116 it(
117 `${name}: exit ${exitCode}, empty stdout, cause on stderr`,
118 async () => {
119 rig.fake.faults.inject({ route: SEARCH, fault });
120 const result = await rig.run(SEARCH_SLACK);
121
122 expect(result.exitCode).toBe(exitCode);
123 expect(result.stdout).toBe('');
124 for (const text of stderr) expect(clean(result.stderr)).toContain(text);
125 expect(searchRequests(result)).toHaveLength(attempts);
126 },
127 TEST_TIMEOUT_MS
128 );
129 }
130
131 const transient: ReadonlyArray<{ name: string; fault: Fault }> = [
132 { name: 'one HTTP 500', fault: { kind: 'status', status: 500 } },
133 {
134 name: 'one HTTP 429 with Retry-After',
135 fault: { kind: 'status', status: 429, headers: { 'retry-after': '1' } },
136 },
137 { name: 'one connection reset', fault: { kind: 'reset' } },
138 ];
139
140 for (const { name, fault } of transient) {
141 it(
142 `${name}: the retry heals it and the caller sees a normal result`,
143 async () => {
144 rig.fake.faults.inject({ route: SEARCH, fault, times: 1 });
145 const result = await rig.run(SEARCH_SLACK);
146
147 expect(result.exitCode).toBe(0);
148 expect(result.stderr).toBe('');
149 expect(stdoutJson(result, SearchResult).results[0]?.primary_tool_slugs).toEqual([
150 'SLACK_SEND_MESSAGE',
151 ]);
152 expect(searchRequests(result).map(entry => entry.status)).toEqual([
153 fault.kind === 'status' ? fault.status : 0,
154 200,
155 ]);
156 },
157 TEST_TIMEOUT_MS
158 );
159 }
160
161 it(
162 'a slow response: waits for it instead of giving up',
163 async () => {
164 rig.fake.faults.inject({ route: SEARCH, fault: { kind: 'delay', ms: 4000 } });
165 const result = await rig.run(SEARCH_SLACK);
166 expect(result.exitCode).toBe(0);
167 expect(searchRequests(result)).toHaveLength(1);
168 expect(stdoutJson(result, SearchResult).results).toHaveLength(1);
169 },
170 TEST_TIMEOUT_MS
171 );
172
173 for (const { name, fault } of [
174 {
175 name: 'valid JSON of an unexpected shape',
176 fault: { kind: 'schema-drift', transform: () => ({ unexpected: true }) },
177 },
178 { name: 'an empty 200 body', fault: { kind: 'empty-body' } },
179 ] satisfies ReadonlyArray<{ name: string; fault: Fault }>) {
180 it(
181 `${name}: reported as an unexpected response`,
182 async () => {
183 rig.fake.faults.inject({ route: SEARCH, fault });
184 const result = await rig.run(SEARCH_SLACK);
185 expect(result.exitCode).toBe(1);
186 expect(result.stdout).toBe('');
187 expect(clean(result.stderr)).not.toContain('TypeError');
188 expect(clean(result.stderr)).toContain('composio upgrade');
189 expect(clean(result.stderr)).toContain(
190 'is not in the shape this version of the CLI expects'
191 );
192 },
193 TEST_TIMEOUT_MS
194 );
195 }
196 });
197
198 // A 429 refused the execution, so it is safe to send again; after any other fault the tool may
199 // have run, so the result says to check first.
200 const EXECUTION_FAULTS: ReadonlyArray<{
201 name: string;
202 fault: Fault;
203 exitCode: number;
204 retry: 'safe' | 'check-first';
205 }> = [
206 { name: 'HTTP 500', fault: { kind: 'status', status: 500 }, exitCode: 1, retry: 'check-first' },
207 {
208 name: 'HTTP 429',
209 fault: { kind: 'status', status: 429, headers: { 'retry-after': '1' } },
210 exitCode: TEMPORARY_FAILURE,
211 retry: 'safe',
212 },
213 {
214 name: 'an HTML error page',
215 fault: { kind: 'html', status: 502 },
216 exitCode: 1,
217 retry: 'check-first',
218 },
219 {
220 name: 'a body that is not JSON',
221 fault: { kind: 'malformed-json' },
222 exitCode: 1,
223 retry: 'check-first',
224 },
225 {
226 name: 'connection reset before any response',
227 fault: { kind: 'reset' },
228 exitCode: 1,
229 retry: 'check-first',
230 },
231 {
232 name: 'connection reset in the middle of the body',
233 fault: { kind: 'reset-mid-body' },
234 exitCode: 1,
235 retry: 'check-first',
236 },
237 ];
238
239 for (const transport of [
240 { name: 'Dashboard', dashboard: true, route: 'dashboard.execute' },
241 { name: 'backend', dashboard: false, route: 'toolRouter.session.execute' },
242 ] as const) {
243 describe(`network faults on a tool execution (${transport.name} transport)`, () => {
244 let rig: CliRig;
245
246 beforeAll(async () => {
247 rig = await startCliRig({
248 label: `net-exec-${transport.name.toLowerCase()}`,
249 dashboard: transport.dashboard,
250 });
251 await rig.login();
252 rig.fake.world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
253 }, RIG_TIMEOUT_MS);
254
255 afterEach(() => rig.fake.faults.clear());
256
257 afterAll(async () => {
258 await writeDebugLog(rig, import.meta.dir, `net-exec-${transport.name.toLowerCase()}`);
259 await rig.stop();
260 });
261
262 it(
263 'without a fault the execution goes through this transport once',
264 async () => {
265 const result = await rig.run(EXECUTE_SLACK);
266 expect(result.exitCode).toBe(0);
267 expect(stdoutJson(result, ExecutionResult).successful).toBe(true);
268 expect(result.requests.filter(entry => entry.route === transport.route)).toHaveLength(1);
269 },
270 TEST_TIMEOUT_MS
271 );
272
273 for (const { name, fault, exitCode, retry } of EXECUTION_FAULTS) {
274 it(
275 `${name}: sent exactly once, reported as a failed result on stdout with retry "${retry}"`,
276 async () => {
277 rig.fake.faults.inject({ route: transport.route, fault });
278 const result = await rig.run(EXECUTE_SLACK);
279
280 expect(result.exitCode).toBe(exitCode);
281 expect(result.requests.filter(entry => entry.route === transport.route)).toHaveLength(1);
282 const outcome = stdoutJson(result, ExecutionResult);
283 expect(outcome.successful).toBe(false);
284 expect(outcome.error ?? '').not.toBe('');
285 expect(outcome.data).toBeUndefined();
286 expect(outcome.retry).toBe(retry);
287 },
288 TEST_TIMEOUT_MS
289 );
290 }
291 });
292 }