Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Network faults: what the CLI does when a host it depends on misbehaves. |
| 3 | * |
| 4 | * Two invariants carry most of the weight. A read may be retried, and either |
| 5 | * heals or fails with exit 1, empty stdout, and the cause on stderr. A tool |
| 6 | * execution is sent exactly once whatever happens, because a second attempt |
| 7 | * could act twice. |
| 8 | * |
| 9 | * A failure that is safe to send again as it is (a 429, which the API refused |
| 10 | * before acting) exits 75 instead of 1, so a caller can retry on the code alone. |
| 11 | * A failed execution says in its result whether it is safe to run again or may |
| 12 | * have run (`retry`). |
| 13 | */ |
| 14 | |
| 15 | import { afterAll, afterEach, beforeAll, describe, expect, it } from 'bun:test'; |
| 16 | import { CONSUMER_PROJECT, CONSUMER_USER_ID, type Fault } from '@e2e-tests/cli-fake-backend'; |
| 17 | import { startCliRig, type CliRig } from '@e2e-tests/cli-fake-backend/rig'; |
| 18 | import { |
| 19 | clean, |
| 20 | EXECUTE_SLACK, |
| 21 | ExecutionResult, |
| 22 | RIG_TIMEOUT_MS, |
| 23 | SEARCH_SLACK, |
| 24 | SearchResult, |
| 25 | stdoutJson, |
| 26 | TEST_TIMEOUT_MS, |
| 27 | writeDebugLog, |
| 28 | } from './support'; |
| 29 | |
| 30 | const SEARCH = 'toolRouter.session.search'; |
| 31 | |
| 32 | /** The exit code of a failure that is safe to send again as it is (`EX_TEMPFAIL`). */ |
| 33 | const TEMPORARY_FAILURE = 75; |
| 34 | const RETRY_HINT = 'run the same command again'; |
| 35 | |
| 36 | describe('network faults on a read (`search`)', () => { |
| 37 | let rig: CliRig; |
| 38 | |
| 39 | beforeAll(async () => { |
| 40 | rig = await startCliRig({ label: 'net-read' }); |
| 41 | await rig.login(); |
| 42 | }, RIG_TIMEOUT_MS); |
| 43 | |
| 44 | afterEach(() => rig.fake.faults.clear()); |
| 45 | |
| 46 | afterAll(async () => { |
| 47 | await writeDebugLog(rig, import.meta.dir, 'net-read'); |
| 48 | await rig.stop(); |
| 49 | }); |
| 50 | |
| 51 | const searchRequests = (result: Awaited<ReturnType<CliRig['run']>>) => |
| 52 | result.requests.filter(entry => entry.route === SEARCH); |
| 53 | |
| 54 | it( |
| 55 | 'connection refused: fails with the cause and an empty stdout', |
| 56 | async () => { |
| 57 | // Port 9 (discard) has no listener on the host. |
| 58 | const result = await rig.run( |
| 59 | `COMPOSIO_BASE_URL=http://host.docker.internal:9 ${SEARCH_SLACK}` |
| 60 | ); |
| 61 | expect(result.exitCode).toBe(1); |
| 62 | expect(result.stdout).toBe(''); |
| 63 | expect(clean(result.stderr)).toContain('ECONNREFUSED'); |
| 64 | expect(result.requests).toEqual([]); |
| 65 | }, |
| 66 | TEST_TIMEOUT_MS |
| 67 | ); |
| 68 | |
| 69 | const persistent: ReadonlyArray<{ |
| 70 | name: string; |
| 71 | fault: Fault; |
| 72 | stderr: ReadonlyArray<string>; |
| 73 | attempts: number; |
| 74 | exitCode?: number; |
| 75 | }> = [ |
| 76 | { |
| 77 | name: 'HTTP 500 on every attempt', |
| 78 | fault: { kind: 'status', status: 500 }, |
| 79 | stderr: ['Injected HTTP 500'], |
| 80 | attempts: 3, |
| 81 | }, |
| 82 | { |
| 83 | name: 'HTTP 429 on every attempt', |
| 84 | fault: { kind: 'status', status: 429, headers: { 'retry-after': '1' } }, |
| 85 | stderr: ['Injected HTTP 429', RETRY_HINT], |
| 86 | attempts: 3, |
| 87 | exitCode: TEMPORARY_FAILURE, |
| 88 | }, |
| 89 | { |
| 90 | name: 'an HTML error page from a proxy', |
| 91 | fault: { kind: 'html', status: 502 }, |
| 92 | stderr: ['502'], |
| 93 | attempts: 3, |
| 94 | }, |
| 95 | { |
| 96 | name: 'connection reset before any response', |
| 97 | fault: { kind: 'reset' }, |
| 98 | stderr: ['ECONNRESET'], |
| 99 | attempts: 3, |
| 100 | }, |
| 101 | { |
| 102 | name: 'connection reset in the middle of the body', |
| 103 | fault: { kind: 'reset-mid-body' }, |
| 104 | stderr: ['ECONNRESET'], |
| 105 | attempts: 1, |
| 106 | }, |
| 107 | { |
| 108 | name: 'a body that is not JSON', |
| 109 | fault: { kind: 'malformed-json' }, |
| 110 | stderr: ['is not in the shape this version of the CLI expects', 'composio upgrade'], |
| 111 | attempts: 1, |
| 112 | }, |
| 113 | ]; |
| 114 | |
| 115 | for (const { name, fault, stderr, attempts, exitCode = 1 } of persistent) { |
| 116 | it( |
| 117 | `${name}: exit ${exitCode}, empty stdout, cause on stderr`, |
| 118 | async () => { |
| 119 | rig.fake.faults.inject({ route: SEARCH, fault }); |
| 120 | const result = await rig.run(SEARCH_SLACK); |
| 121 | |
| 122 | expect(result.exitCode).toBe(exitCode); |
| 123 | expect(result.stdout).toBe(''); |
| 124 | for (const text of stderr) expect(clean(result.stderr)).toContain(text); |
| 125 | expect(searchRequests(result)).toHaveLength(attempts); |
| 126 | }, |
| 127 | TEST_TIMEOUT_MS |
| 128 | ); |
| 129 | } |
| 130 | |
| 131 | const transient: ReadonlyArray<{ name: string; fault: Fault }> = [ |
| 132 | { name: 'one HTTP 500', fault: { kind: 'status', status: 500 } }, |
| 133 | { |
| 134 | name: 'one HTTP 429 with Retry-After', |
| 135 | fault: { kind: 'status', status: 429, headers: { 'retry-after': '1' } }, |
| 136 | }, |
| 137 | { name: 'one connection reset', fault: { kind: 'reset' } }, |
| 138 | ]; |
| 139 | |
| 140 | for (const { name, fault } of transient) { |
| 141 | it( |
| 142 | `${name}: the retry heals it and the caller sees a normal result`, |
| 143 | async () => { |
| 144 | rig.fake.faults.inject({ route: SEARCH, fault, times: 1 }); |
| 145 | const result = await rig.run(SEARCH_SLACK); |
| 146 | |
| 147 | expect(result.exitCode).toBe(0); |
| 148 | expect(result.stderr).toBe(''); |
| 149 | expect(stdoutJson(result, SearchResult).results[0]?.primary_tool_slugs).toEqual([ |
| 150 | 'SLACK_SEND_MESSAGE', |
| 151 | ]); |
| 152 | expect(searchRequests(result).map(entry => entry.status)).toEqual([ |
| 153 | fault.kind === 'status' ? fault.status : 0, |
| 154 | 200, |
| 155 | ]); |
| 156 | }, |
| 157 | TEST_TIMEOUT_MS |
| 158 | ); |
| 159 | } |
| 160 | |
| 161 | it( |
| 162 | 'a slow response: waits for it instead of giving up', |
| 163 | async () => { |
| 164 | rig.fake.faults.inject({ route: SEARCH, fault: { kind: 'delay', ms: 4000 } }); |
| 165 | const result = await rig.run(SEARCH_SLACK); |
| 166 | expect(result.exitCode).toBe(0); |
| 167 | expect(searchRequests(result)).toHaveLength(1); |
| 168 | expect(stdoutJson(result, SearchResult).results).toHaveLength(1); |
| 169 | }, |
| 170 | TEST_TIMEOUT_MS |
| 171 | ); |
| 172 | |
| 173 | for (const { name, fault } of [ |
| 174 | { |
| 175 | name: 'valid JSON of an unexpected shape', |
| 176 | fault: { kind: 'schema-drift', transform: () => ({ unexpected: true }) }, |
| 177 | }, |
| 178 | { name: 'an empty 200 body', fault: { kind: 'empty-body' } }, |
| 179 | ] satisfies ReadonlyArray<{ name: string; fault: Fault }>) { |
| 180 | it( |
| 181 | `${name}: reported as an unexpected response`, |
| 182 | async () => { |
| 183 | rig.fake.faults.inject({ route: SEARCH, fault }); |
| 184 | const result = await rig.run(SEARCH_SLACK); |
| 185 | expect(result.exitCode).toBe(1); |
| 186 | expect(result.stdout).toBe(''); |
| 187 | expect(clean(result.stderr)).not.toContain('TypeError'); |
| 188 | expect(clean(result.stderr)).toContain('composio upgrade'); |
| 189 | expect(clean(result.stderr)).toContain( |
| 190 | 'is not in the shape this version of the CLI expects' |
| 191 | ); |
| 192 | }, |
| 193 | TEST_TIMEOUT_MS |
| 194 | ); |
| 195 | } |
| 196 | }); |
| 197 | |
| 198 | // A 429 refused the execution, so it is safe to send again; after any other fault the tool may |
| 199 | // have run, so the result says to check first. |
| 200 | const EXECUTION_FAULTS: ReadonlyArray<{ |
| 201 | name: string; |
| 202 | fault: Fault; |
| 203 | exitCode: number; |
| 204 | retry: 'safe' | 'check-first'; |
| 205 | }> = [ |
| 206 | { name: 'HTTP 500', fault: { kind: 'status', status: 500 }, exitCode: 1, retry: 'check-first' }, |
| 207 | { |
| 208 | name: 'HTTP 429', |
| 209 | fault: { kind: 'status', status: 429, headers: { 'retry-after': '1' } }, |
| 210 | exitCode: TEMPORARY_FAILURE, |
| 211 | retry: 'safe', |
| 212 | }, |
| 213 | { |
| 214 | name: 'an HTML error page', |
| 215 | fault: { kind: 'html', status: 502 }, |
| 216 | exitCode: 1, |
| 217 | retry: 'check-first', |
| 218 | }, |
| 219 | { |
| 220 | name: 'a body that is not JSON', |
| 221 | fault: { kind: 'malformed-json' }, |
| 222 | exitCode: 1, |
| 223 | retry: 'check-first', |
| 224 | }, |
| 225 | { |
| 226 | name: 'connection reset before any response', |
| 227 | fault: { kind: 'reset' }, |
| 228 | exitCode: 1, |
| 229 | retry: 'check-first', |
| 230 | }, |
| 231 | { |
| 232 | name: 'connection reset in the middle of the body', |
| 233 | fault: { kind: 'reset-mid-body' }, |
| 234 | exitCode: 1, |
| 235 | retry: 'check-first', |
| 236 | }, |
| 237 | ]; |
| 238 | |
| 239 | for (const transport of [ |
| 240 | { name: 'Dashboard', dashboard: true, route: 'dashboard.execute' }, |
| 241 | { name: 'backend', dashboard: false, route: 'toolRouter.session.execute' }, |
| 242 | ] as const) { |
| 243 | describe(`network faults on a tool execution (${transport.name} transport)`, () => { |
| 244 | let rig: CliRig; |
| 245 | |
| 246 | beforeAll(async () => { |
| 247 | rig = await startCliRig({ |
| 248 | label: `net-exec-${transport.name.toLowerCase()}`, |
| 249 | dashboard: transport.dashboard, |
| 250 | }); |
| 251 | await rig.login(); |
| 252 | rig.fake.world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 253 | }, RIG_TIMEOUT_MS); |
| 254 | |
| 255 | afterEach(() => rig.fake.faults.clear()); |
| 256 | |
| 257 | afterAll(async () => { |
| 258 | await writeDebugLog(rig, import.meta.dir, `net-exec-${transport.name.toLowerCase()}`); |
| 259 | await rig.stop(); |
| 260 | }); |
| 261 | |
| 262 | it( |
| 263 | 'without a fault the execution goes through this transport once', |
| 264 | async () => { |
| 265 | const result = await rig.run(EXECUTE_SLACK); |
| 266 | expect(result.exitCode).toBe(0); |
| 267 | expect(stdoutJson(result, ExecutionResult).successful).toBe(true); |
| 268 | expect(result.requests.filter(entry => entry.route === transport.route)).toHaveLength(1); |
| 269 | }, |
| 270 | TEST_TIMEOUT_MS |
| 271 | ); |
| 272 | |
| 273 | for (const { name, fault, exitCode, retry } of EXECUTION_FAULTS) { |
| 274 | it( |
| 275 | `${name}: sent exactly once, reported as a failed result on stdout with retry "${retry}"`, |
| 276 | async () => { |
| 277 | rig.fake.faults.inject({ route: transport.route, fault }); |
| 278 | const result = await rig.run(EXECUTE_SLACK); |
| 279 | |
| 280 | expect(result.exitCode).toBe(exitCode); |
| 281 | expect(result.requests.filter(entry => entry.route === transport.route)).toHaveLength(1); |
| 282 | const outcome = stdoutJson(result, ExecutionResult); |
| 283 | expect(outcome.successful).toBe(false); |
| 284 | expect(outcome.error ?? '').not.toBe(''); |
| 285 | expect(outcome.data).toBeUndefined(); |
| 286 | expect(outcome.retry).toBe(retry); |
| 287 | }, |
| 288 | TEST_TIMEOUT_MS |
| 289 | ); |
| 290 | } |
| 291 | }); |
| 292 | } |