Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 395 lines 13.8 KB
Raw
1 /**
2 * Grader mutation, without a container: the scripted path plays what the
3 * hand-authored file says, a dropped command keeps the rest of the path, and
4 * each verdict follows from made-up evidence.
5 */
6
7 import { describe, expect, it } from 'bun:test';
8 import {
9 CONSUMER_PROJECT,
10 CONSUMER_USER_ID,
11 DEVELOPER_PROJECT,
12 World,
13 type JournalEntry,
14 } from '@e2e-tests/cli-fake-backend';
15 import { runAgent, type ModelAdapter, type ModelTurn, type ShellOutcome } from './src/agent';
16 import { fixturePath, HAND_AUTHORED } from './src/fixtures';
17 import type { Check, Evidence, Journey } from './src/journeys';
18 import {
19 exitCodeOf,
20 failuresOf,
21 grade,
22 outcomeOf,
23 renderReports,
24 reportJourney,
25 verdictOf,
26 } from './src/mutation';
27 import { readScriptedPath, scriptedAdapter, scriptedCommands } from './src/scripted';
28
29 const OK: ShellOutcome = { exitCode: 0, stdout: '', stderr: '' };
30
31 /** Plays an adapter against a shell that answers every command with `OK`. */
32 const play = (
33 adapter: ModelAdapter,
34 hear?: (message: string, waiting: boolean) => string | undefined
35 ) => {
36 const ran: string[] = [];
37 return runAgent({
38 adapter,
39 maxTurns: 12,
40 hear,
41 execute: async command => {
42 ran.push(command);
43 return OK;
44 },
45 }).then(transcript => ({ transcript, ran }));
46 };
47
48 const turns: ReadonlyArray<ModelTurn> = [
49 { text: '', calls: [{ id: 'a', command: 'composio whoami' }] },
50 { text: 'Open the page, I will wait.', calls: [{ id: 'b', command: 'composio login --poll' }] },
51 { text: '', calls: [{ id: 'c', command: 'composio whoami' }] },
52 { text: 'Logged in.', calls: [] },
53 ];
54
55 describe('the scripted path', () => {
56 it('reads the hand-authored file turn by turn, shell calls only', () => {
57 const path = readScriptedPath(fixturePath('first-run-login', 'openai', HAND_AUTHORED));
58 expect(scriptedCommands(path)).toEqual([
59 'composio whoami',
60 'composio login',
61 'composio login --poll',
62 'composio whoami',
63 ]);
64 expect(path.at(-1)).toEqual({
65 text: 'The CLI is now logged in as dev@acme-e2e.test.',
66 calls: [],
67 });
68 });
69
70 it('plays every command, then finishes on the last turn', async () => {
71 const { transcript, ran } = await play(scriptedAdapter(turns));
72 expect(ran).toEqual(['composio whoami', 'composio login --poll', 'composio whoami']);
73 expect(transcript.stop).toBe('finished');
74 expect(transcript.modelTurns).toBe(4);
75 });
76
77 it('leaves out the dropped command and plays the turns after it', async () => {
78 const { transcript, ran } = await play(scriptedAdapter(turns, { drop: 1 }));
79 expect(ran).toEqual(['composio whoami', 'composio whoami']);
80 expect(transcript.messages).toEqual(['Open the page, I will wait.', 'Logged in.']);
81 expect(transcript.stop).toBe('finished');
82 expect(transcript.modelTurns).toBe(4);
83 });
84
85 it('hands the person the text of a turn whose command was dropped', async () => {
86 const heard: Array<[string, boolean]> = [];
87 await play(scriptedAdapter(turns, { drop: 1 }), (message, waiting) => {
88 heard.push([message, waiting]);
89 return undefined;
90 });
91 expect(heard[1]).toEqual(['Open the page, I will wait.', true]);
92 });
93
94 it('stops as the script ending when the person answers its last turn', async () => {
95 const { transcript, ran } = await play(scriptedAdapter(turns), () => 'Thanks.');
96 expect(ran).toEqual(['composio whoami', 'composio login --poll', 'composio whoami']);
97 expect(transcript.stop).toEqual({ provider: 'script-ended' });
98 });
99 });
100
101 describe('runAgent with an adapter that does not say whether more turns follow', () => {
102 // A model's turn without a command is its last, as before `hasMore` existed.
103 it('stops at the first turn that asks for no command', async () => {
104 let asked = 0;
105 const adapter: ModelAdapter = {
106 provider: 'openai',
107 model: 'test',
108 next: async () => {
109 asked += 1;
110 return asked === 1 ? { text: 'Nothing to do.', calls: [] } : turns[0]!;
111 },
112 record: () => {},
113 say: () => {},
114 };
115 const { transcript, ran } = await play(adapter);
116 expect(asked).toBe(1);
117 expect(ran).toEqual([]);
118 expect(transcript.stop).toBe('finished');
119 });
120 });
121
122 /** A made-up journal entry: a request that changed nothing, unless `fields` say otherwise. */
123 const entry = (fields: Partial<JournalEntry>): JournalEntry => ({
124 seq: 1,
125 role: 'backend',
126 method: 'POST',
127 path: '/',
128 query: {},
129 route: null,
130 keyClass: 'user',
131 keyKnown: true,
132 orgId: null,
133 project: CONSUMER_PROJECT.label,
134 requestedProject: null,
135 projectFrom: null,
136 userId: null,
137 body: null,
138 status: 200,
139 effect: null,
140 tool: null,
141 fault: null,
142 ...fields,
143 });
144
145 // Made-up evidence: what the verdicts read is the checks and the outcome.
146 const evidenceWith = (world: World, requests: ReadonlyArray<JournalEntry> = []): Evidence => ({
147 transcript: { commands: [], messages: [], replies: [], stop: 'finished', modelTurns: 1 },
148 world,
149 requests,
150 commandRequests: [],
151 startingConnections: [],
152 whoamiAfter: OK,
153 });
154
155 const journeyChecking = (checks: (run: Evidence) => ReadonlyArray<Check>): Journey => ({
156 id: 'made-up',
157 title: 'made up',
158 prompt: '',
159 split: 'train',
160 setup: async () => {},
161 checks,
162 });
163
164 /** Passes when a login session exists and is linked; says nothing about anything else. */
165 const loggedIn = journeyChecking(run => [
166 { name: 'a login session was linked', ok: run.world.loginSessions[0]?.status === 'linked' },
167 ]);
168
169 const linkedWorld = () => {
170 const world = new World();
171 world.linkLoginSession(world.createLoginSession().id);
172 return world;
173 };
174
175 describe('verdicts', () => {
176 const unmutated = grade(loggedIn, evidenceWith(linkedWorld()));
177
178 it('calls a step needed when a check fails without it', () => {
179 const mutated = grade(loggedIn, evidenceWith(new World()));
180 expect(verdictOf(unmutated, mutated)).toBe('needed');
181 });
182
183 it('calls a step unused when every check passes and the outcome is the same', () => {
184 const mutated = grade(loggedIn, evidenceWith(linkedWorld()));
185 expect(verdictOf(unmutated, mutated)).toBe('unused');
186 });
187
188 it('calls a check too weak when every check passes but the run ended elsewhere', () => {
189 // The same login, plus a Slack account connected that the unmutated run never connected.
190 const world = linkedWorld();
191 world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
192 const mutated = grade(loggedIn, evidenceWith(world));
193 expect(mutated.checks.every(check => check.ok)).toBe(true);
194 expect(verdictOf(unmutated, mutated)).toBe('check-too-weak');
195 });
196
197 it('leaves the connections the setup made out of the outcome', () => {
198 const world = new World();
199 const seeded = world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
200 expect(outcomeOf({ ...evidenceWith(world), startingConnections: [seeded.id] })).toBe(
201 outcomeOf(evidenceWith(new World()))
202 );
203 });
204
205 it('counts a tool that ran in the outcome, and a request that changed nothing not', () => {
206 const quiet = outcomeOf(evidenceWith(new World()));
207 expect(outcomeOf(evidenceWith(new World(), [entry({ route: 'tools.list' })]))).toBe(quiet);
208 expect(
209 outcomeOf(
210 evidenceWith(new World(), [
211 entry({ effect: 'tool-executed', tool: { slug: 'SLACK_SEND_MESSAGE', arguments: {} } }),
212 ])
213 )
214 ).not.toBe(quiet);
215 });
216 // Every field the outcome compares, one at a time: if `outcomeOf` stopped
217 // reading one, a run that ended elsewhere in that way would be called `unused`.
218 describe('an outcome that differs in one thing', () => {
219 const sentToGeneral = (fields: Partial<JournalEntry> = {}) =>
220 entry({
221 effect: 'tool-executed',
222 userId: CONSUMER_USER_ID,
223 tool: { slug: 'SLACK_SEND_MESSAGE', arguments: { channel: 'general' } },
224 ...fields,
225 });
226 const ended = (
227 options: {
228 login?: 'pending' | 'linked';
229 connection?: 'begun' | 'connected';
230 execution?: JournalEntry;
231 whoamiAfter?: ShellOutcome;
232 } = {}
233 ): Evidence => {
234 const world = new World();
235 const session = world.createLoginSession();
236 if ((options.login ?? 'linked') === 'linked') world.linkLoginSession(session.id);
237 if ((options.connection ?? 'connected') === 'connected') {
238 world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
239 } else {
240 world.beginConnection({
241 projectLabel: CONSUMER_PROJECT.label,
242 userId: CONSUMER_USER_ID,
243 toolkit: 'slack',
244 });
245 }
246 return {
247 ...evidenceWith(world, [options.execution ?? sentToGeneral()]),
248 whoamiAfter: options.whoamiAfter ?? OK,
249 };
250 };
251
252 it('is the same outcome when nothing differs', () => {
253 expect(outcomeOf(ended())).toBe(outcomeOf(ended()));
254 });
255
256 const differences: ReadonlyArray<[string, Evidence]> = [
257 ['the login was never linked', ended({ login: 'pending' })],
258 [
259 'the next whoami finds no login',
260 ended({ whoamiAfter: { exitCode: 1, stdout: '', stderr: 'not logged in' } }),
261 ],
262 ['the connection was begun but never authorized', ended({ connection: 'begun' })],
263 [
264 'the tool ran with other arguments',
265 ended({
266 execution: sentToGeneral({
267 tool: { slug: 'SLACK_SEND_MESSAGE', arguments: { channel: 'random' } },
268 }),
269 }),
270 ],
271 [
272 'the tool ran on another project',
273 ended({ execution: sentToGeneral({ project: DEVELOPER_PROJECT.label }) }),
274 ],
275 [
276 'the tool ran as another user',
277 ended({ execution: sentToGeneral({ userId: 'someone-else' }) }),
278 ],
279 ];
280
281 for (const [name, run] of differences) {
282 it(`is another outcome when ${name}`, () => {
283 expect(outcomeOf(run)).not.toBe(outcomeOf(ended()));
284 });
285 }
286 });
287 });
288
289 describe('reports', () => {
290 const unmutated = grade(loggedIn, evidenceWith(linkedWorld()));
291 const weakWorld = linkedWorld();
292 weakWorld.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
293 const report = reportJourney({
294 journey: 'made-up',
295 fixture: 'fixtures/made-up.openai.hand-authored.json',
296 unmutated,
297 mutated: [
298 { command: 'composio login', graded: grade(loggedIn, evidenceWith(new World())) },
299 { command: 'composio whoami', graded: grade(loggedIn, evidenceWith(linkedWorld())) },
300 { command: 'composio link slack', graded: grade(loggedIn, evidenceWith(weakWorld)) },
301 ],
302 });
303
304 it('gives one row per dropped command, numbered from 1, with the checks on each side', () => {
305 expect(report.unmutated).toEqual({ ok: true, failed: [] });
306 expect(report.steps).toEqual([
307 {
308 step: 1,
309 command: 'composio login',
310 verdict: 'needed',
311 passed: [],
312 failed: ['a login session was linked'],
313 },
314 {
315 step: 2,
316 command: 'composio whoami',
317 verdict: 'unused',
318 passed: ['a login session was linked'],
319 failed: [],
320 },
321 {
322 step: 3,
323 command: 'composio link slack',
324 verdict: 'check-too-weak',
325 passed: ['a login session was linked'],
326 failed: [],
327 },
328 ]);
329 });
330
331 // `report` has one step of each verdict: needed, unused, check-too-weak.
332 it('exits 0 by default, whatever the verdicts are', () => {
333 expect(exitCodeOf([report], false)).toBe(0);
334 expect(failuresOf([report], false)).toEqual([]);
335 });
336
337 it('exits 1 with --strict on a check-too-weak step, and says which', () => {
338 expect(exitCodeOf([report], true)).toBe(1);
339 expect(failuresOf([report], true)).toEqual([
340 'made-up: step 3 (composio link slack) is check-too-weak',
341 ]);
342 });
343
344 // A read, a `--help`, or a command that fails on purpose changes nothing for
345 // an outcome check to notice, so `unused` is not a grader defect.
346 it('exits 0 with --strict when the only steps that are not needed are unused', () => {
347 const withoutWeak = { ...report, steps: report.steps.slice(0, 2) };
348 expect(exitCodeOf([withoutWeak], true)).toBe(0);
349 expect(failuresOf([withoutWeak], true)).toEqual([]);
350 });
351
352 it('exits 0 with --strict when every step is needed', () => {
353 expect(exitCodeOf([{ ...report, steps: report.steps.slice(0, 1) }], true)).toBe(0);
354 });
355
356 it('renders a row per step and a total', () => {
357 const table = renderReports([report]);
358 expect(table).toContain('unmutated path pass');
359 expect(table).toMatch(/^1 +composio login +needed +0\/1$/m);
360 expect(table).toContain('failed: a login session was linked');
361 expect(table).toMatch(/^3 +composio link slack +check-too-weak +1\/1$/m);
362 expect(table).toContain('3 step(s): 1 needed, 1 unused, 1 check-too-weak');
363 });
364
365 describe('when the unmutated path fails a check', () => {
366 const broken = reportJourney({
367 journey: 'made-up',
368 fixture: 'fixtures/made-up.openai.hand-authored.json',
369 unmutated: grade(loggedIn, evidenceWith(new World())),
370 mutated: [
371 { command: 'composio whoami', graded: grade(loggedIn, evidenceWith(linkedWorld())) },
372 ],
373 });
374
375 it('reports no step, since no verdict would mean anything', () => {
376 expect(broken.unmutated).toEqual({ ok: false, failed: ['a login session was linked'] });
377 expect(broken.steps).toEqual([]);
378 });
379
380 it('exits 1 with or without --strict', () => {
381 expect(exitCodeOf([report, broken], false)).toBe(1);
382 expect(exitCodeOf([broken], true)).toBe(1);
383 });
384
385 it('names the journey whose path broke, in both modes', () => {
386 expect(failuresOf([broken], false)).toEqual(['made-up: the unmutated path failed a check']);
387 expect(failuresOf([broken], true)).toEqual(['made-up: the unmutated path failed a check']);
388 });
389
390 it('says so in the table', () => {
391 expect(renderReports([broken])).toContain('unmutated path FAILED; no step was dropped');
392 expect(renderReports([broken])).toContain('1 unmutated path(s) FAILED');
393 });
394 });
395 });