Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Grader mutation, without a container: the scripted path plays what the |
| 3 | * hand-authored file says, a dropped command keeps the rest of the path, and |
| 4 | * each verdict follows from made-up evidence. |
| 5 | */ |
| 6 | |
| 7 | import { describe, expect, it } from 'bun:test'; |
| 8 | import { |
| 9 | CONSUMER_PROJECT, |
| 10 | CONSUMER_USER_ID, |
| 11 | DEVELOPER_PROJECT, |
| 12 | World, |
| 13 | type JournalEntry, |
| 14 | } from '@e2e-tests/cli-fake-backend'; |
| 15 | import { runAgent, type ModelAdapter, type ModelTurn, type ShellOutcome } from './src/agent'; |
| 16 | import { fixturePath, HAND_AUTHORED } from './src/fixtures'; |
| 17 | import type { Check, Evidence, Journey } from './src/journeys'; |
| 18 | import { |
| 19 | exitCodeOf, |
| 20 | failuresOf, |
| 21 | grade, |
| 22 | outcomeOf, |
| 23 | renderReports, |
| 24 | reportJourney, |
| 25 | verdictOf, |
| 26 | } from './src/mutation'; |
| 27 | import { readScriptedPath, scriptedAdapter, scriptedCommands } from './src/scripted'; |
| 28 | |
| 29 | const OK: ShellOutcome = { exitCode: 0, stdout: '', stderr: '' }; |
| 30 | |
| 31 | /** Plays an adapter against a shell that answers every command with `OK`. */ |
| 32 | const play = ( |
| 33 | adapter: ModelAdapter, |
| 34 | hear?: (message: string, waiting: boolean) => string | undefined |
| 35 | ) => { |
| 36 | const ran: string[] = []; |
| 37 | return runAgent({ |
| 38 | adapter, |
| 39 | maxTurns: 12, |
| 40 | hear, |
| 41 | execute: async command => { |
| 42 | ran.push(command); |
| 43 | return OK; |
| 44 | }, |
| 45 | }).then(transcript => ({ transcript, ran })); |
| 46 | }; |
| 47 | |
| 48 | const turns: ReadonlyArray<ModelTurn> = [ |
| 49 | { text: '', calls: [{ id: 'a', command: 'composio whoami' }] }, |
| 50 | { text: 'Open the page, I will wait.', calls: [{ id: 'b', command: 'composio login --poll' }] }, |
| 51 | { text: '', calls: [{ id: 'c', command: 'composio whoami' }] }, |
| 52 | { text: 'Logged in.', calls: [] }, |
| 53 | ]; |
| 54 | |
| 55 | describe('the scripted path', () => { |
| 56 | it('reads the hand-authored file turn by turn, shell calls only', () => { |
| 57 | const path = readScriptedPath(fixturePath('first-run-login', 'openai', HAND_AUTHORED)); |
| 58 | expect(scriptedCommands(path)).toEqual([ |
| 59 | 'composio whoami', |
| 60 | 'composio login', |
| 61 | 'composio login --poll', |
| 62 | 'composio whoami', |
| 63 | ]); |
| 64 | expect(path.at(-1)).toEqual({ |
| 65 | text: 'The CLI is now logged in as dev@acme-e2e.test.', |
| 66 | calls: [], |
| 67 | }); |
| 68 | }); |
| 69 | |
| 70 | it('plays every command, then finishes on the last turn', async () => { |
| 71 | const { transcript, ran } = await play(scriptedAdapter(turns)); |
| 72 | expect(ran).toEqual(['composio whoami', 'composio login --poll', 'composio whoami']); |
| 73 | expect(transcript.stop).toBe('finished'); |
| 74 | expect(transcript.modelTurns).toBe(4); |
| 75 | }); |
| 76 | |
| 77 | it('leaves out the dropped command and plays the turns after it', async () => { |
| 78 | const { transcript, ran } = await play(scriptedAdapter(turns, { drop: 1 })); |
| 79 | expect(ran).toEqual(['composio whoami', 'composio whoami']); |
| 80 | expect(transcript.messages).toEqual(['Open the page, I will wait.', 'Logged in.']); |
| 81 | expect(transcript.stop).toBe('finished'); |
| 82 | expect(transcript.modelTurns).toBe(4); |
| 83 | }); |
| 84 | |
| 85 | it('hands the person the text of a turn whose command was dropped', async () => { |
| 86 | const heard: Array<[string, boolean]> = []; |
| 87 | await play(scriptedAdapter(turns, { drop: 1 }), (message, waiting) => { |
| 88 | heard.push([message, waiting]); |
| 89 | return undefined; |
| 90 | }); |
| 91 | expect(heard[1]).toEqual(['Open the page, I will wait.', true]); |
| 92 | }); |
| 93 | |
| 94 | it('stops as the script ending when the person answers its last turn', async () => { |
| 95 | const { transcript, ran } = await play(scriptedAdapter(turns), () => 'Thanks.'); |
| 96 | expect(ran).toEqual(['composio whoami', 'composio login --poll', 'composio whoami']); |
| 97 | expect(transcript.stop).toEqual({ provider: 'script-ended' }); |
| 98 | }); |
| 99 | }); |
| 100 | |
| 101 | describe('runAgent with an adapter that does not say whether more turns follow', () => { |
| 102 | // A model's turn without a command is its last, as before `hasMore` existed. |
| 103 | it('stops at the first turn that asks for no command', async () => { |
| 104 | let asked = 0; |
| 105 | const adapter: ModelAdapter = { |
| 106 | provider: 'openai', |
| 107 | model: 'test', |
| 108 | next: async () => { |
| 109 | asked += 1; |
| 110 | return asked === 1 ? { text: 'Nothing to do.', calls: [] } : turns[0]!; |
| 111 | }, |
| 112 | record: () => {}, |
| 113 | say: () => {}, |
| 114 | }; |
| 115 | const { transcript, ran } = await play(adapter); |
| 116 | expect(asked).toBe(1); |
| 117 | expect(ran).toEqual([]); |
| 118 | expect(transcript.stop).toBe('finished'); |
| 119 | }); |
| 120 | }); |
| 121 | |
| 122 | /** A made-up journal entry: a request that changed nothing, unless `fields` say otherwise. */ |
| 123 | const entry = (fields: Partial<JournalEntry>): JournalEntry => ({ |
| 124 | seq: 1, |
| 125 | role: 'backend', |
| 126 | method: 'POST', |
| 127 | path: '/', |
| 128 | query: {}, |
| 129 | route: null, |
| 130 | keyClass: 'user', |
| 131 | keyKnown: true, |
| 132 | orgId: null, |
| 133 | project: CONSUMER_PROJECT.label, |
| 134 | requestedProject: null, |
| 135 | projectFrom: null, |
| 136 | userId: null, |
| 137 | body: null, |
| 138 | status: 200, |
| 139 | effect: null, |
| 140 | tool: null, |
| 141 | fault: null, |
| 142 | ...fields, |
| 143 | }); |
| 144 | |
| 145 | // Made-up evidence: what the verdicts read is the checks and the outcome. |
| 146 | const evidenceWith = (world: World, requests: ReadonlyArray<JournalEntry> = []): Evidence => ({ |
| 147 | transcript: { commands: [], messages: [], replies: [], stop: 'finished', modelTurns: 1 }, |
| 148 | world, |
| 149 | requests, |
| 150 | commandRequests: [], |
| 151 | startingConnections: [], |
| 152 | whoamiAfter: OK, |
| 153 | }); |
| 154 | |
| 155 | const journeyChecking = (checks: (run: Evidence) => ReadonlyArray<Check>): Journey => ({ |
| 156 | id: 'made-up', |
| 157 | title: 'made up', |
| 158 | prompt: '', |
| 159 | split: 'train', |
| 160 | setup: async () => {}, |
| 161 | checks, |
| 162 | }); |
| 163 | |
| 164 | /** Passes when a login session exists and is linked; says nothing about anything else. */ |
| 165 | const loggedIn = journeyChecking(run => [ |
| 166 | { name: 'a login session was linked', ok: run.world.loginSessions[0]?.status === 'linked' }, |
| 167 | ]); |
| 168 | |
| 169 | const linkedWorld = () => { |
| 170 | const world = new World(); |
| 171 | world.linkLoginSession(world.createLoginSession().id); |
| 172 | return world; |
| 173 | }; |
| 174 | |
| 175 | describe('verdicts', () => { |
| 176 | const unmutated = grade(loggedIn, evidenceWith(linkedWorld())); |
| 177 | |
| 178 | it('calls a step needed when a check fails without it', () => { |
| 179 | const mutated = grade(loggedIn, evidenceWith(new World())); |
| 180 | expect(verdictOf(unmutated, mutated)).toBe('needed'); |
| 181 | }); |
| 182 | |
| 183 | it('calls a step unused when every check passes and the outcome is the same', () => { |
| 184 | const mutated = grade(loggedIn, evidenceWith(linkedWorld())); |
| 185 | expect(verdictOf(unmutated, mutated)).toBe('unused'); |
| 186 | }); |
| 187 | |
| 188 | it('calls a check too weak when every check passes but the run ended elsewhere', () => { |
| 189 | // The same login, plus a Slack account connected that the unmutated run never connected. |
| 190 | const world = linkedWorld(); |
| 191 | world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 192 | const mutated = grade(loggedIn, evidenceWith(world)); |
| 193 | expect(mutated.checks.every(check => check.ok)).toBe(true); |
| 194 | expect(verdictOf(unmutated, mutated)).toBe('check-too-weak'); |
| 195 | }); |
| 196 | |
| 197 | it('leaves the connections the setup made out of the outcome', () => { |
| 198 | const world = new World(); |
| 199 | const seeded = world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 200 | expect(outcomeOf({ ...evidenceWith(world), startingConnections: [seeded.id] })).toBe( |
| 201 | outcomeOf(evidenceWith(new World())) |
| 202 | ); |
| 203 | }); |
| 204 | |
| 205 | it('counts a tool that ran in the outcome, and a request that changed nothing not', () => { |
| 206 | const quiet = outcomeOf(evidenceWith(new World())); |
| 207 | expect(outcomeOf(evidenceWith(new World(), [entry({ route: 'tools.list' })]))).toBe(quiet); |
| 208 | expect( |
| 209 | outcomeOf( |
| 210 | evidenceWith(new World(), [ |
| 211 | entry({ effect: 'tool-executed', tool: { slug: 'SLACK_SEND_MESSAGE', arguments: {} } }), |
| 212 | ]) |
| 213 | ) |
| 214 | ).not.toBe(quiet); |
| 215 | }); |
| 216 | // Every field the outcome compares, one at a time: if `outcomeOf` stopped |
| 217 | // reading one, a run that ended elsewhere in that way would be called `unused`. |
| 218 | describe('an outcome that differs in one thing', () => { |
| 219 | const sentToGeneral = (fields: Partial<JournalEntry> = {}) => |
| 220 | entry({ |
| 221 | effect: 'tool-executed', |
| 222 | userId: CONSUMER_USER_ID, |
| 223 | tool: { slug: 'SLACK_SEND_MESSAGE', arguments: { channel: 'general' } }, |
| 224 | ...fields, |
| 225 | }); |
| 226 | const ended = ( |
| 227 | options: { |
| 228 | login?: 'pending' | 'linked'; |
| 229 | connection?: 'begun' | 'connected'; |
| 230 | execution?: JournalEntry; |
| 231 | whoamiAfter?: ShellOutcome; |
| 232 | } = {} |
| 233 | ): Evidence => { |
| 234 | const world = new World(); |
| 235 | const session = world.createLoginSession(); |
| 236 | if ((options.login ?? 'linked') === 'linked') world.linkLoginSession(session.id); |
| 237 | if ((options.connection ?? 'connected') === 'connected') { |
| 238 | world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 239 | } else { |
| 240 | world.beginConnection({ |
| 241 | projectLabel: CONSUMER_PROJECT.label, |
| 242 | userId: CONSUMER_USER_ID, |
| 243 | toolkit: 'slack', |
| 244 | }); |
| 245 | } |
| 246 | return { |
| 247 | ...evidenceWith(world, [options.execution ?? sentToGeneral()]), |
| 248 | whoamiAfter: options.whoamiAfter ?? OK, |
| 249 | }; |
| 250 | }; |
| 251 | |
| 252 | it('is the same outcome when nothing differs', () => { |
| 253 | expect(outcomeOf(ended())).toBe(outcomeOf(ended())); |
| 254 | }); |
| 255 | |
| 256 | const differences: ReadonlyArray<[string, Evidence]> = [ |
| 257 | ['the login was never linked', ended({ login: 'pending' })], |
| 258 | [ |
| 259 | 'the next whoami finds no login', |
| 260 | ended({ whoamiAfter: { exitCode: 1, stdout: '', stderr: 'not logged in' } }), |
| 261 | ], |
| 262 | ['the connection was begun but never authorized', ended({ connection: 'begun' })], |
| 263 | [ |
| 264 | 'the tool ran with other arguments', |
| 265 | ended({ |
| 266 | execution: sentToGeneral({ |
| 267 | tool: { slug: 'SLACK_SEND_MESSAGE', arguments: { channel: 'random' } }, |
| 268 | }), |
| 269 | }), |
| 270 | ], |
| 271 | [ |
| 272 | 'the tool ran on another project', |
| 273 | ended({ execution: sentToGeneral({ project: DEVELOPER_PROJECT.label }) }), |
| 274 | ], |
| 275 | [ |
| 276 | 'the tool ran as another user', |
| 277 | ended({ execution: sentToGeneral({ userId: 'someone-else' }) }), |
| 278 | ], |
| 279 | ]; |
| 280 | |
| 281 | for (const [name, run] of differences) { |
| 282 | it(`is another outcome when ${name}`, () => { |
| 283 | expect(outcomeOf(run)).not.toBe(outcomeOf(ended())); |
| 284 | }); |
| 285 | } |
| 286 | }); |
| 287 | }); |
| 288 | |
| 289 | describe('reports', () => { |
| 290 | const unmutated = grade(loggedIn, evidenceWith(linkedWorld())); |
| 291 | const weakWorld = linkedWorld(); |
| 292 | weakWorld.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack'); |
| 293 | const report = reportJourney({ |
| 294 | journey: 'made-up', |
| 295 | fixture: 'fixtures/made-up.openai.hand-authored.json', |
| 296 | unmutated, |
| 297 | mutated: [ |
| 298 | { command: 'composio login', graded: grade(loggedIn, evidenceWith(new World())) }, |
| 299 | { command: 'composio whoami', graded: grade(loggedIn, evidenceWith(linkedWorld())) }, |
| 300 | { command: 'composio link slack', graded: grade(loggedIn, evidenceWith(weakWorld)) }, |
| 301 | ], |
| 302 | }); |
| 303 | |
| 304 | it('gives one row per dropped command, numbered from 1, with the checks on each side', () => { |
| 305 | expect(report.unmutated).toEqual({ ok: true, failed: [] }); |
| 306 | expect(report.steps).toEqual([ |
| 307 | { |
| 308 | step: 1, |
| 309 | command: 'composio login', |
| 310 | verdict: 'needed', |
| 311 | passed: [], |
| 312 | failed: ['a login session was linked'], |
| 313 | }, |
| 314 | { |
| 315 | step: 2, |
| 316 | command: 'composio whoami', |
| 317 | verdict: 'unused', |
| 318 | passed: ['a login session was linked'], |
| 319 | failed: [], |
| 320 | }, |
| 321 | { |
| 322 | step: 3, |
| 323 | command: 'composio link slack', |
| 324 | verdict: 'check-too-weak', |
| 325 | passed: ['a login session was linked'], |
| 326 | failed: [], |
| 327 | }, |
| 328 | ]); |
| 329 | }); |
| 330 | |
| 331 | // `report` has one step of each verdict: needed, unused, check-too-weak. |
| 332 | it('exits 0 by default, whatever the verdicts are', () => { |
| 333 | expect(exitCodeOf([report], false)).toBe(0); |
| 334 | expect(failuresOf([report], false)).toEqual([]); |
| 335 | }); |
| 336 | |
| 337 | it('exits 1 with --strict on a check-too-weak step, and says which', () => { |
| 338 | expect(exitCodeOf([report], true)).toBe(1); |
| 339 | expect(failuresOf([report], true)).toEqual([ |
| 340 | 'made-up: step 3 (composio link slack) is check-too-weak', |
| 341 | ]); |
| 342 | }); |
| 343 | |
| 344 | // A read, a `--help`, or a command that fails on purpose changes nothing for |
| 345 | // an outcome check to notice, so `unused` is not a grader defect. |
| 346 | it('exits 0 with --strict when the only steps that are not needed are unused', () => { |
| 347 | const withoutWeak = { ...report, steps: report.steps.slice(0, 2) }; |
| 348 | expect(exitCodeOf([withoutWeak], true)).toBe(0); |
| 349 | expect(failuresOf([withoutWeak], true)).toEqual([]); |
| 350 | }); |
| 351 | |
| 352 | it('exits 0 with --strict when every step is needed', () => { |
| 353 | expect(exitCodeOf([{ ...report, steps: report.steps.slice(0, 1) }], true)).toBe(0); |
| 354 | }); |
| 355 | |
| 356 | it('renders a row per step and a total', () => { |
| 357 | const table = renderReports([report]); |
| 358 | expect(table).toContain('unmutated path pass'); |
| 359 | expect(table).toMatch(/^1 +composio login +needed +0\/1$/m); |
| 360 | expect(table).toContain('failed: a login session was linked'); |
| 361 | expect(table).toMatch(/^3 +composio link slack +check-too-weak +1\/1$/m); |
| 362 | expect(table).toContain('3 step(s): 1 needed, 1 unused, 1 check-too-weak'); |
| 363 | }); |
| 364 | |
| 365 | describe('when the unmutated path fails a check', () => { |
| 366 | const broken = reportJourney({ |
| 367 | journey: 'made-up', |
| 368 | fixture: 'fixtures/made-up.openai.hand-authored.json', |
| 369 | unmutated: grade(loggedIn, evidenceWith(new World())), |
| 370 | mutated: [ |
| 371 | { command: 'composio whoami', graded: grade(loggedIn, evidenceWith(linkedWorld())) }, |
| 372 | ], |
| 373 | }); |
| 374 | |
| 375 | it('reports no step, since no verdict would mean anything', () => { |
| 376 | expect(broken.unmutated).toEqual({ ok: false, failed: ['a login session was linked'] }); |
| 377 | expect(broken.steps).toEqual([]); |
| 378 | }); |
| 379 | |
| 380 | it('exits 1 with or without --strict', () => { |
| 381 | expect(exitCodeOf([report, broken], false)).toBe(1); |
| 382 | expect(exitCodeOf([broken], true)).toBe(1); |
| 383 | }); |
| 384 | |
| 385 | it('names the journey whose path broke, in both modes', () => { |
| 386 | expect(failuresOf([broken], false)).toEqual(['made-up: the unmutated path failed a check']); |
| 387 | expect(failuresOf([broken], true)).toEqual(['made-up: the unmutated path failed a check']); |
| 388 | }); |
| 389 | |
| 390 | it('says so in the table', () => { |
| 391 | expect(renderReports([broken])).toContain('unmutated path FAILED; no step was dropped'); |
| 392 | expect(renderReports([broken])).toContain('1 unmutated path(s) FAILED'); |
| 393 | }); |
| 394 | }); |
| 395 | }); |