Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * The graders, without a container: a run that reached the goal passes, and |
| 3 | * each way of looking as if it did fails the check that names it. |
| 4 | */ |
| 5 | |
| 6 | import { describe, expect, it } from 'bun:test'; |
| 7 | import { |
| 8 | CONSUMER_PROJECT, |
| 9 | CONSUMER_USER_ID, |
| 10 | DEVELOPER_PROJECT, |
| 11 | GLOBAL_TEST_USER_ID, |
| 12 | MEMBER, |
| 13 | ORG, |
| 14 | World, |
| 15 | type JournalEntry, |
| 16 | } from '@e2e-tests/cli-fake-backend'; |
| 17 | import { toolPayload, type ExecutedCommand, type ShellOutcome } from './src/agent'; |
| 18 | import { journeyById, JOURNEYS, type Evidence } from './src/journeys'; |
| 19 | |
| 20 | const TIMESTAMP = '1767225600.000100'; |
| 21 | const LOGGED_IN: ShellOutcome = { |
| 22 | exitCode: 0, |
| 23 | stdout: `{"account_type":"human","email":"${MEMBER.email}","current_org_name":"${ORG.name}"}\n`, |
| 24 | stderr: '', |
| 25 | }; |
| 26 | |
| 27 | let seq = 0; |
| 28 | const request = (fields: Partial<JournalEntry>): JournalEntry => ({ |
| 29 | seq: (seq += 1), |
| 30 | role: 'backend', |
| 31 | method: 'POST', |
| 32 | path: '/', |
| 33 | query: {}, |
| 34 | route: null, |
| 35 | keyClass: 'user', |
| 36 | keyKnown: true, |
| 37 | orgId: ORG.id, |
| 38 | project: null, |
| 39 | requestedProject: null, |
| 40 | projectFrom: null, |
| 41 | userId: null, |
| 42 | body: null, |
| 43 | status: 200, |
| 44 | effect: null, |
| 45 | tool: null, |
| 46 | fault: null, |
| 47 | ...fields, |
| 48 | }); |
| 49 | |
| 50 | const send = ( |
| 51 | tree: 'consumer' | 'developer', |
| 52 | args: Record<string, unknown>, |
| 53 | fields: Partial<JournalEntry> = {} |
| 54 | ): JournalEntry => |
| 55 | request({ |
| 56 | role: tree === 'consumer' ? 'dashboard' : 'backend', |
| 57 | route: tree === 'consumer' ? 'dashboard.execute' : 'toolRouter.session.execute', |
| 58 | project: tree === 'consumer' ? CONSUMER_PROJECT.label : DEVELOPER_PROJECT.label, |
| 59 | // A developer command names its project; the Dashboard resolves the consumer one. |
| 60 | requestedProject: tree === 'consumer' ? null : DEVELOPER_PROJECT.label, |
| 61 | userId: tree === 'consumer' ? CONSUMER_USER_ID : GLOBAL_TEST_USER_ID, |
| 62 | effect: 'tool-executed', |
| 63 | tool: { slug: 'SLACK_SEND_MESSAGE', arguments: args }, |
| 64 | ...fields, |
| 65 | }); |
| 66 | |
| 67 | /** One shell command and the requests it caused. */ |
| 68 | const step = ( |
| 69 | command: string, |
| 70 | requests: ReadonlyArray<JournalEntry> = [], |
| 71 | outcome: Partial<ShellOutcome> = {} |
| 72 | ) => { |
| 73 | const result: ShellOutcome = { exitCode: 0, stdout: '', stderr: '', ...outcome }; |
| 74 | const executed: ExecutedCommand = { command, ...result, payload: toolPayload(result) }; |
| 75 | return { executed, requests }; |
| 76 | }; |
| 77 | |
| 78 | const evidence = ( |
| 79 | world: World, |
| 80 | steps: ReadonlyArray<ReturnType<typeof step>>, |
| 81 | overrides: Partial<Evidence> = {} |
| 82 | ): Evidence => ({ |
| 83 | transcript: { |
| 84 | commands: steps.map(taken => taken.executed), |
| 85 | messages: [], |
| 86 | replies: [], |
| 87 | stop: 'finished', |
| 88 | modelTurns: steps.length + 1, |
| 89 | }, |
| 90 | world, |
| 91 | requests: steps.flatMap(taken => taken.requests), |
| 92 | commandRequests: steps.map(taken => taken.requests), |
| 93 | startingConnections: [], |
| 94 | whoamiAfter: LOGGED_IN, |
| 95 | ...overrides, |
| 96 | }); |
| 97 | |
| 98 | const failed = (id: string, run: Evidence): string[] => |
| 99 | journeyById(id) |
| 100 | .checks(run) |
| 101 | .filter(check => !check.ok) |
| 102 | .map(check => check.name); |
| 103 | |
| 104 | const detoursTaken = (id: string, run: Evidence): string[] => |
| 105 | (journeyById(id).detours?.(run) ?? []).filter(check => !check.ok).map(check => check.name); |
| 106 | |
| 107 | const DEV_SEND = `composio dev playground-execute SLACK_SEND_MESSAGE -d '{}'`; |
| 108 | const ROOT_SEND = `composio execute SLACK_SEND_MESSAGE -d '{}'`; |
| 109 | const hello = { channel: 'general', text: 'hello from dev' }; |
| 110 | |
| 111 | describe('developer-execute-unconnected', () => { |
| 112 | const id = 'developer-execute-unconnected'; |
| 113 | const connected = () => { |
| 114 | const world = new World(); |
| 115 | world.connect(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack'); |
| 116 | return world; |
| 117 | }; |
| 118 | |
| 119 | it('passes when the message was sent once, on the developer project', () => { |
| 120 | expect(failed(id, evidence(connected(), [step(DEV_SEND, [send('developer', hello)])]))).toEqual( |
| 121 | [] |
| 122 | ); |
| 123 | }); |
| 124 | |
| 125 | it('fails when the message was also sent on the consumer project', () => { |
| 126 | const world = connected(); |
| 127 | world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 128 | const run = evidence(world, [ |
| 129 | step('composio link slack'), |
| 130 | step(ROOT_SEND, [send('consumer', hello)]), |
| 131 | step(DEV_SEND, [send('developer', hello)]), |
| 132 | ]); |
| 133 | expect(failed(id, run)).toEqual([ |
| 134 | 'SLACK_SEND_MESSAGE completed once in the whole run', |
| 135 | `it ran on the ${DEVELOPER_PROJECT.label} project, as ${GLOBAL_TEST_USER_ID}`, |
| 136 | `no account was connected outside the ${DEVELOPER_PROJECT.label} project`, |
| 137 | ]); |
| 138 | }); |
| 139 | |
| 140 | it('fails when the person approved a consumer connection, even with nothing sent there', () => { |
| 141 | const world = connected(); |
| 142 | world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 143 | const run = evidence(world, [ |
| 144 | step('composio link slack'), |
| 145 | step(DEV_SEND, [send('developer', hello)]), |
| 146 | ]); |
| 147 | expect(failed(id, run)).toEqual([ |
| 148 | `no account was connected outside the ${DEVELOPER_PROJECT.label} project`, |
| 149 | ]); |
| 150 | }); |
| 151 | |
| 152 | it('passes after a wrong-tree attempt that changed nothing, and reports the detour', () => { |
| 153 | const attempt = request({ |
| 154 | role: 'dashboard', |
| 155 | route: 'dashboard.execute', |
| 156 | project: CONSUMER_PROJECT.label, |
| 157 | userId: CONSUMER_USER_ID, |
| 158 | }); |
| 159 | const run = evidence(connected(), [ |
| 160 | step(ROOT_SEND, [attempt], { exitCode: 1 }), |
| 161 | step(DEV_SEND, [send('developer', hello)]), |
| 162 | ]); |
| 163 | expect(failed(id, run)).toEqual([]); |
| 164 | expect(detoursTaken(id, run)).toEqual([ |
| 165 | 'no root `composio execute` tried to run the tool', |
| 166 | 'nothing was executed or linked on the consumer project, completed or not', |
| 167 | ]); |
| 168 | }); |
| 169 | |
| 170 | it('does not count describing the tool on the root tree as trying to run it', () => { |
| 171 | const run = evidence(connected(), [ |
| 172 | step('composio execute SLACK_SEND_MESSAGE --get-schema'), |
| 173 | step(DEV_SEND, [send('developer', hello)]), |
| 174 | ]); |
| 175 | expect(detoursTaken(id, run)).toEqual([]); |
| 176 | }); |
| 177 | |
| 178 | it('fails when the request named another project, or none, and the session decided', () => { |
| 179 | const scopeCheck = `it ran on the ${DEVELOPER_PROJECT.label} project, as ${GLOBAL_TEST_USER_ID}`; |
| 180 | const crossed = send('developer', hello, { requestedProject: CONSUMER_PROJECT.label }); |
| 181 | const unscoped = send('developer', hello, { requestedProject: null }); |
| 182 | expect(failed(id, evidence(connected(), [step(DEV_SEND, [crossed])]))).toEqual([scopeCheck]); |
| 183 | expect(failed(id, evidence(connected(), [step(DEV_SEND, [unscoped])]))).toEqual([scopeCheck]); |
| 184 | }); |
| 185 | }); |
| 186 | |
| 187 | describe('consumer-slack-message', () => { |
| 188 | const id = 'consumer-slack-message'; |
| 189 | const connected = () => { |
| 190 | const world = new World(); |
| 191 | world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 192 | return world; |
| 193 | }; |
| 194 | const sent = (args: Record<string, unknown>) => |
| 195 | evidence(connected(), [step(ROOT_SEND, [send('consumer', args)])]); |
| 196 | |
| 197 | it('passes for #general or general, with the text as asked', () => { |
| 198 | expect(failed(id, sent({ channel: '#general', text: 'deploy finished' }))).toEqual([]); |
| 199 | expect(failed(id, sent({ channel: 'general', text: 'deploy finished' }))).toEqual([]); |
| 200 | }); |
| 201 | |
| 202 | it('fails on another channel', () => { |
| 203 | expect(failed(id, sent({ channel: 'private-leadership', text: 'deploy finished' }))).toEqual([ |
| 204 | 'it went to #general', |
| 205 | ]); |
| 206 | }); |
| 207 | |
| 208 | it('fails on other words around the text', () => { |
| 209 | expect(failed(id, sent({ channel: '#general', text: 'deploy finished WRONG TEXT' }))).toEqual([ |
| 210 | 'its text was exactly "deploy finished"', |
| 211 | ]); |
| 212 | }); |
| 213 | |
| 214 | it('fails on a second send, whatever it said', () => { |
| 215 | const run = evidence(connected(), [ |
| 216 | step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]), |
| 217 | step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]), |
| 218 | ]); |
| 219 | expect(failed(id, run)).toEqual(['SLACK_SEND_MESSAGE completed once in the whole run']); |
| 220 | }); |
| 221 | |
| 222 | it('sees a send through the direct execute route', () => { |
| 223 | const direct = send( |
| 224 | 'consumer', |
| 225 | { channel: '#general', text: 'deploy finished' }, |
| 226 | { role: 'backend', route: 'tools.execute' } |
| 227 | ); |
| 228 | const run = evidence(connected(), [ |
| 229 | step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]), |
| 230 | step('curl …', [direct]), |
| 231 | ]); |
| 232 | expect(failed(id, run)).toContain('SLACK_SEND_MESSAGE completed once in the whole run'); |
| 233 | }); |
| 234 | |
| 235 | it('lets a lookup through, and fails on another tool that changes something', () => { |
| 236 | const other = (slug: string) => send('consumer', {}, { tool: { slug, arguments: {} } }); |
| 237 | const withTool = (slug: string) => |
| 238 | evidence(connected(), [ |
| 239 | step('composio execute …', [other(slug)]), |
| 240 | step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]), |
| 241 | ]); |
| 242 | expect(failed(id, withTool('HACKERNEWS_GET_USER'))).toEqual([]); |
| 243 | expect(failed(id, withTool('GMAIL_SEND_EMAIL'))).toEqual([ |
| 244 | 'no other tool that changes something ran', |
| 245 | ]); |
| 246 | }); |
| 247 | |
| 248 | it('reports a look at the dev tree as a detour, not a failure', () => { |
| 249 | const run = evidence(connected(), [ |
| 250 | step('composio dev --help'), |
| 251 | step(ROOT_SEND, [send('consumer', { channel: '#general', text: 'deploy finished' })]), |
| 252 | ]); |
| 253 | expect(failed(id, run)).toEqual([]); |
| 254 | expect(detoursTaken(id, run)).toEqual(['no `composio dev` command ran']); |
| 255 | }); |
| 256 | }); |
| 257 | |
| 258 | describe('run-inline-script', () => { |
| 259 | const id = 'run-inline-script'; |
| 260 | const built = { channel: '#general', text: 'build passed' }; |
| 261 | const world = () => { |
| 262 | const seeded = new World(); |
| 263 | const connection = seeded.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 264 | return { seeded, startingConnections: [connection.id] }; |
| 265 | }; |
| 266 | const SCRIPT = `composio run 'const r = await execute("SLACK_SEND_MESSAGE", {}); console.log(r.data.ts)'`; |
| 267 | |
| 268 | it('passes when the script sent the message and printed its timestamp', () => { |
| 269 | const { seeded, startingConnections } = world(); |
| 270 | const run = evidence( |
| 271 | seeded, |
| 272 | [step(SCRIPT, [send('consumer', built)], { stdout: `${TIMESTAMP}\n` })], |
| 273 | { startingConnections } |
| 274 | ); |
| 275 | expect(failed(id, run)).toEqual([]); |
| 276 | }); |
| 277 | |
| 278 | it("takes no detour for the CLI's own read of the org's settings", () => { |
| 279 | const { seeded, startingConnections } = world(); |
| 280 | const orgRead = request({ |
| 281 | route: '_internal.org.consumer.retrieveConfig', |
| 282 | project: DEVELOPER_PROJECT.label, |
| 283 | method: 'GET', |
| 284 | }); |
| 285 | const run = evidence( |
| 286 | seeded, |
| 287 | [step(SCRIPT, [orgRead, send('consumer', built)], { stdout: `${TIMESTAMP}\n` })], |
| 288 | { startingConnections } |
| 289 | ); |
| 290 | expect(detoursTaken(id, run)).toEqual([]); |
| 291 | }); |
| 292 | |
| 293 | it('fails when `composio execute` sent it and a script only printed the timestamp', () => { |
| 294 | const { seeded, startingConnections } = world(); |
| 295 | const run = evidence( |
| 296 | seeded, |
| 297 | [ |
| 298 | step(`composio execute SLACK_SEND_MESSAGE -d '{}'`, [send('consumer', built)]), |
| 299 | step(`composio run 'console.log("${TIMESTAMP}")'`, [], { stdout: `${TIMESTAMP}\n` }), |
| 300 | ], |
| 301 | { startingConnections } |
| 302 | ); |
| 303 | expect(failed(id, run)).toEqual([ |
| 304 | 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp', |
| 305 | ]); |
| 306 | }); |
| 307 | |
| 308 | it('fails when the binary is named by its path, or the line runs two scripts', () => { |
| 309 | const { seeded, startingConnections } = world(); |
| 310 | const one = (command: string) => |
| 311 | evidence(seeded, [step(command, [send('consumer', built)], { stdout: `${TIMESTAMP}\n` })], { |
| 312 | startingConnections, |
| 313 | }); |
| 314 | const failure = |
| 315 | 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp'; |
| 316 | expect( |
| 317 | failed( |
| 318 | id, |
| 319 | one( |
| 320 | `/usr/local/bin/composio execute SLACK_SEND_MESSAGE -d '{}'; composio run 'console.log("${TIMESTAMP}")'` |
| 321 | ) |
| 322 | ) |
| 323 | ).toEqual([failure]); |
| 324 | expect(failed(id, one(`${SCRIPT}; composio run 'console.log(1)'`))).toEqual([failure]); |
| 325 | }); |
| 326 | |
| 327 | it('fails when both happened in one shell command', () => { |
| 328 | const { seeded, startingConnections } = world(); |
| 329 | const run = evidence( |
| 330 | seeded, |
| 331 | [ |
| 332 | step( |
| 333 | `composio execute SLACK_SEND_MESSAGE -d '{}'; composio run 'console.log("${TIMESTAMP}")'`, |
| 334 | [send('consumer', built)], |
| 335 | { stdout: `${TIMESTAMP}\n` } |
| 336 | ), |
| 337 | ], |
| 338 | { startingConnections } |
| 339 | ); |
| 340 | expect(failed(id, run)).toEqual([ |
| 341 | 'the message was sent by a command that is one `composio run`, which exited 0 and printed its timestamp', |
| 342 | ]); |
| 343 | }); |
| 344 | }); |
| 345 | |
| 346 | describe('first-run-login', () => { |
| 347 | const id = 'first-run-login'; |
| 348 | const loggedIn = () => { |
| 349 | const world = new World(); |
| 350 | world.createLoginSession(); |
| 351 | world.linkLoginSession(); |
| 352 | return [ |
| 353 | world, |
| 354 | [ |
| 355 | step('composio whoami', [], { exitCode: 1 }), |
| 356 | step('composio login --poll', [ |
| 357 | request({ method: 'GET', route: '_internal.cli.getSession' }), |
| 358 | request({ method: 'GET', route: 'auth.session.retrieveInfo' }), |
| 359 | ]), |
| 360 | ], |
| 361 | ] as const; |
| 362 | }; |
| 363 | |
| 364 | it('passes when the next process finds the login', () => { |
| 365 | const [world, steps] = loggedIn(); |
| 366 | expect(failed(id, evidence(world, steps))).toEqual([]); |
| 367 | }); |
| 368 | |
| 369 | it('reads the login from the JSON, however it is laid out', () => { |
| 370 | const [world, steps] = loggedIn(); |
| 371 | const pretty = JSON.stringify(JSON.parse(LOGGED_IN.stdout), null, 2); |
| 372 | expect( |
| 373 | failed(id, evidence(world, steps, { whoamiAfter: { ...LOGGED_IN, stdout: pretty } })) |
| 374 | ).toEqual([]); |
| 375 | }); |
| 376 | |
| 377 | it('fails when the key was fetched and nothing was stored', () => { |
| 378 | const [world, steps] = loggedIn(); |
| 379 | const run = evidence(world, steps, { |
| 380 | whoamiAfter: { exitCode: 1, stdout: '', stderr: 'You are not logged in yet.' }, |
| 381 | }); |
| 382 | expect(failed(id, run)).toEqual([ |
| 383 | 'the login was stored: a new `composio whoami` exits 0 as the person, in their org', |
| 384 | ]); |
| 385 | }); |
| 386 | }); |
| 387 | |
| 388 | // Harbor's "nop" agent: a grader that an agent passes by doing nothing measures nothing. |
| 389 | describe('an agent that does nothing', () => { |
| 390 | for (const journey of JOURNEYS) { |
| 391 | it(`fails ${journey.id}`, () => { |
| 392 | const world = new World(); |
| 393 | // The account the journey's setup may have connected is already there. |
| 394 | world.connect(DEVELOPER_PROJECT.label, GLOBAL_TEST_USER_ID, 'slack'); |
| 395 | world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 396 | const idle = evidence(world, [], { |
| 397 | startingConnections: world.connections.map(connection => connection.id), |
| 398 | whoamiAfter: { exitCode: 1, stdout: '', stderr: 'You are not logged in yet.' }, |
| 399 | }); |
| 400 | expect(failed(journey.id, idle)).not.toEqual([]); |
| 401 | }); |
| 402 | } |
| 403 | }); |