1
+/**
2
+ * Grader mutation, without a container: the scripted path plays what the
3
+ * hand-authored file says, a dropped command keeps the rest of the path, and
4
+ * each verdict follows from made-up evidence.
5
+ */
6
+
7
+import { describe, expect, it } from 'bun:test';
8
+import {
9
+ CONSUMER_PROJECT,
10
+ CONSUMER_USER_ID,
11
+ DEVELOPER_PROJECT,
12
+ World,
13
+ type JournalEntry,
14
+} from '@e2e-tests/cli-fake-backend';
15
+import { runAgent, type ModelAdapter, type ModelTurn, type ShellOutcome } from './src/agent';
16
+import { fixturePath, HAND_AUTHORED } from './src/fixtures';
17
+import type { Check, Evidence, Journey } from './src/journeys';
18
+import {
19
+ exitCodeOf,
20
+ grade,
21
+ outcomeOf,
22
+ renderReports,
23
+ reportJourney,
24
+ verdictOf,
25
+} from './src/mutation';
26
+import { readScriptedPath, scriptedAdapter, scriptedCommands } from './src/scripted';
27
+
28
+const OK: ShellOutcome = { exitCode: 0, stdout: '', stderr: '' };
29
+
30
+/** Plays an adapter against a shell that answers every command with `OK`. */
31
+const play = (
32
+ adapter: ModelAdapter,
33
+ hear?: (message: string, waiting: boolean) => string | undefined
34
+) => {
35
+ const ran: string[] = [];
36
+ return runAgent({
37
+ adapter,
38
+ maxTurns: 12,
39
+ hear,
40
+ execute: async command => {
41
+ ran.push(command);
42
+ return OK;
43
+ },
44
+ }).then(transcript => ({ transcript, ran }));
45
+};
46
+
47
+const turns: ReadonlyArray<ModelTurn> = [
48
+ { text: '', calls: [{ id: 'a', command: 'composio whoami' }] },
49
+ { text: 'Open the page, I will wait.', calls: [{ id: 'b', command: 'composio login --poll' }] },
50
+ { text: '', calls: [{ id: 'c', command: 'composio whoami' }] },
51
+ { text: 'Logged in.', calls: [] },
52
+];
53
+
54
+describe('the scripted path', () => {
55
+ it('reads the hand-authored file turn by turn, shell calls only', () => {
56
+ const path = readScriptedPath(fixturePath('first-run-login', 'openai', HAND_AUTHORED));
57
+ expect(scriptedCommands(path)).toEqual([
58
+ 'composio whoami',
59
+ 'composio login',
60
+ 'composio login --poll',
61
+ 'composio whoami',
62
+ ]);
63
+ expect(path.at(-1)).toEqual({
64
+ text: 'The CLI is now logged in as dev@acme-e2e.test.',
65
+ calls: [],
66
+ });
67
+ });
68
+
69
+ it('plays every command, then finishes on the last turn', async () => {
70
+ const { transcript, ran } = await play(scriptedAdapter(turns));
71
+ expect(ran).toEqual(['composio whoami', 'composio login --poll', 'composio whoami']);
72
+ expect(transcript.stop).toBe('finished');
73
+ expect(transcript.modelTurns).toBe(4);
74
+ });
75
+
76
+ it('leaves out the dropped command and plays the turns after it', async () => {
77
+ const { transcript, ran } = await play(scriptedAdapter(turns, { drop: 1 }));
78
+ expect(ran).toEqual(['composio whoami', 'composio whoami']);
79
+ expect(transcript.messages).toEqual(['Open the page, I will wait.', 'Logged in.']);
80
+ expect(transcript.stop).toBe('finished');
81
+ expect(transcript.modelTurns).toBe(4);
82
+ });
83
+
84
+ it('hands the person the text of a turn whose command was dropped', async () => {
85
+ const heard: Array<[string, boolean]> = [];
86
+ await play(scriptedAdapter(turns, { drop: 1 }), (message, waiting) => {
87
+ heard.push([message, waiting]);
88
+ return undefined;
89
+ });
90
+ expect(heard[1]).toEqual(['Open the page, I will wait.', true]);
91
+ });
92
+
93
+ it('stops as the script ending when the person answers its last turn', async () => {
94
+ const { transcript, ran } = await play(scriptedAdapter(turns), () => 'Thanks.');
95
+ expect(ran).toEqual(['composio whoami', 'composio login --poll', 'composio whoami']);
96
+ expect(transcript.stop).toEqual({ provider: 'script-ended' });
97
+ });
98
+});
99
+
100
+describe('runAgent with an adapter that does not say whether more turns follow', () => {
101
+ // A model's turn without a command is its last, as before `hasMore` existed.
102
+ it('stops at the first turn that asks for no command', async () => {
103
+ let asked = 0;
104
+ const adapter: ModelAdapter = {
105
+ provider: 'openai',
106
+ model: 'test',
107
+ next: async () => {
108
+ asked += 1;
109
+ return asked === 1 ? { text: 'Nothing to do.', calls: [] } : turns[0]!;
110
+ },
111
+ record: () => {},
112
+ say: () => {},
113
+ };
114
+ const { transcript, ran } = await play(adapter);
115
+ expect(asked).toBe(1);
116
+ expect(ran).toEqual([]);
117
+ expect(transcript.stop).toBe('finished');
118
+ });
119
+});
120
+
121
+/** A made-up journal entry: a request that changed nothing, unless `fields` say otherwise. */
122
+const entry = (fields: Partial<JournalEntry>): JournalEntry => ({
123
+ seq: 1,
124
+ role: 'backend',
125
+ method: 'POST',
126
+ path: '/',
127
+ query: {},
128
+ route: null,
129
+ keyClass: 'user',
130
+ keyKnown: true,
131
+ orgId: null,
132
+ project: CONSUMER_PROJECT.label,
133
+ requestedProject: null,
134
+ projectFrom: null,
135
+ userId: null,
136
+ body: null,
137
+ status: 200,
138
+ effect: null,
139
+ tool: null,
140
+ fault: null,
141
+ ...fields,
142
+});
143
+
144
+// Made-up evidence: what the verdicts read is the checks and the outcome.
145
+const evidenceWith = (world: World, requests: ReadonlyArray<JournalEntry> = []): Evidence => ({
146
+ transcript: { commands: [], messages: [], replies: [], stop: 'finished', modelTurns: 1 },
147
+ world,
148
+ requests,
149
+ commandRequests: [],
150
+ startingConnections: [],
151
+ whoamiAfter: OK,
152
+});
153
+
154
+const journeyChecking = (checks: (run: Evidence) => ReadonlyArray<Check>): Journey => ({
155
+ id: 'made-up',
156
+ title: 'made up',
157
+ prompt: '',
158
+ split: 'train',
159
+ setup: async () => {},
160
+ checks,
161
+});
162
+
163
+/** Passes when a login session exists and is linked; says nothing about anything else. */
164
+const loggedIn = journeyChecking(run => [
165
+ { name: 'a login session was linked', ok: run.world.loginSessions[0]?.status === 'linked' },
166
+]);
167
+
168
+const linkedWorld = () => {
169
+ const world = new World();
170
+ world.linkLoginSession(world.createLoginSession().id);
171
+ return world;
172
+};
173
+
174
+describe('verdicts', () => {
175
+ const unmutated = grade(loggedIn, evidenceWith(linkedWorld()));
176
+
177
+ it('calls a step needed when a check fails without it', () => {
178
+ const mutated = grade(loggedIn, evidenceWith(new World()));
179
+ expect(verdictOf(unmutated, mutated)).toBe('needed');
180
+ });
181
+
182
+ it('calls a step unused when every check passes and the outcome is the same', () => {
183
+ const mutated = grade(loggedIn, evidenceWith(linkedWorld()));
184
+ expect(verdictOf(unmutated, mutated)).toBe('unused');
185
+ });
186
+
187
+ it('calls a check too weak when every check passes but the run ended elsewhere', () => {
188
+ // The same login, plus a Slack account connected that the unmutated run never connected.
189
+ const world = linkedWorld();
190
+ world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
191
+ const mutated = grade(loggedIn, evidenceWith(world));
192
+ expect(mutated.checks.every(check => check.ok)).toBe(true);
193
+ expect(verdictOf(unmutated, mutated)).toBe('check-too-weak');
194
+ });
195
+
196
+ it('leaves the connections the setup made out of the outcome', () => {
197
+ const world = new World();
198
+ const seeded = world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
199
+ expect(outcomeOf({ ...evidenceWith(world), startingConnections: [seeded.id] })).toBe(
200
+ outcomeOf(evidenceWith(new World()))
201
+ );
202
+ });
203
+
204
+ it('counts a tool that ran in the outcome, and a request that changed nothing not', () => {
205
+ const quiet = outcomeOf(evidenceWith(new World()));
206
+ expect(outcomeOf(evidenceWith(new World(), [entry({ route: 'tools.list' })]))).toBe(quiet);
207
+ expect(
208
+ outcomeOf(
209
+ evidenceWith(new World(), [
210
+ entry({ effect: 'tool-executed', tool: { slug: 'SLACK_SEND_MESSAGE', arguments: {} } }),
211
+ ])
212
+ )
213
+ ).not.toBe(quiet);
214
+ });
215
+ // Every field the outcome compares, one at a time: if `outcomeOf` stopped
216
+ // reading one, a run that ended elsewhere in that way would be called `unused`.
217
+ describe('an outcome that differs in one thing', () => {
218
+ const sentToGeneral = (fields: Partial<JournalEntry> = {}) =>
219
+ entry({
220
+ effect: 'tool-executed',
221
+ userId: CONSUMER_USER_ID,
222
+ tool: { slug: 'SLACK_SEND_MESSAGE', arguments: { channel: 'general' } },
223
+ ...fields,
224
+ });
225
+ const ended = (
226
+ options: {
227
+ login?: 'pending' | 'linked';
228
+ connection?: 'begun' | 'connected';
229
+ execution?: JournalEntry;
230
+ whoamiAfter?: ShellOutcome;
231
+ } = {}
232
+ ): Evidence => {
233
+ const world = new World();
234
+ const session = world.createLoginSession();
235
+ if ((options.login ?? 'linked') === 'linked') world.linkLoginSession(session.id);
236
+ if ((options.connection ?? 'connected') === 'connected') {
237
+ world.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
238
+ } else {
239
+ world.beginConnection({
240
+ projectLabel: CONSUMER_PROJECT.label,
241
+ userId: CONSUMER_USER_ID,
242
+ toolkit: 'slack',
243
+ });
244
+ }
245
+ return {
246
+ ...evidenceWith(world, [options.execution ?? sentToGeneral()]),
247
+ whoamiAfter: options.whoamiAfter ?? OK,
248
+ };
249
+ };
250
+
251
+ it('is the same outcome when nothing differs', () => {
252
+ expect(outcomeOf(ended())).toBe(outcomeOf(ended()));
253
+ });
254
+
255
+ const differences: ReadonlyArray<[string, Evidence]> = [
256
+ ['the login was never linked', ended({ login: 'pending' })],
257
+ [
258
+ 'the next whoami finds no login',
259
+ ended({ whoamiAfter: { exitCode: 1, stdout: '', stderr: 'not logged in' } }),
260
+ ],
261
+ ['the connection was begun but never authorized', ended({ connection: 'begun' })],
262
+ [
263
+ 'the tool ran with other arguments',
264
+ ended({
265
+ execution: sentToGeneral({
266
+ tool: { slug: 'SLACK_SEND_MESSAGE', arguments: { channel: 'random' } },
267
+ }),
268
+ }),
269
+ ],
270
+ [
271
+ 'the tool ran on another project',
272
+ ended({ execution: sentToGeneral({ project: DEVELOPER_PROJECT.label }) }),
273
+ ],
274
+ [
275
+ 'the tool ran as another user',
276
+ ended({ execution: sentToGeneral({ userId: 'someone-else' }) }),
277
+ ],
278
+ ];
279
+
280
+ for (const [name, run] of differences) {
281
+ it(`is another outcome when ${name}`, () => {
282
+ expect(outcomeOf(run)).not.toBe(outcomeOf(ended()));
283
+ });
284
+ }
285
+ });
286
+});
287
+
288
+describe('reports', () => {
289
+ const unmutated = grade(loggedIn, evidenceWith(linkedWorld()));
290
+ const weakWorld = linkedWorld();
291
+ weakWorld.connect(CONSUMER_PROJECT.label, CONSUMER_USER_ID, 'slack');
292
+ const report = reportJourney({
293
+ journey: 'made-up',
294
+ fixture: 'fixtures/made-up.openai.hand-authored.json',
295
+ unmutated,
296
+ mutated: [
297
+ { command: 'composio login', graded: grade(loggedIn, evidenceWith(new World())) },
298
+ { command: 'composio whoami', graded: grade(loggedIn, evidenceWith(linkedWorld())) },
299
+ { command: 'composio link slack', graded: grade(loggedIn, evidenceWith(weakWorld)) },
300
+ ],
301
+ });
302
+
303
+ it('gives one row per dropped command, numbered from 1, with the checks on each side', () => {
304
+ expect(report.unmutated).toEqual({ ok: true, failed: [] });
305
+ expect(report.steps).toEqual([
306
+ {
307
+ step: 1,
308
+ command: 'composio login',
309
+ verdict: 'needed',
310
+ passed: [],
311
+ failed: ['a login session was linked'],
312
+ },
313
+ {
314
+ step: 2,
315
+ command: 'composio whoami',
316
+ verdict: 'unused',
317
+ passed: ['a login session was linked'],
318
+ failed: [],
319
+ },
320
+ {
321
+ step: 3,
322
+ command: 'composio link slack',
323
+ verdict: 'check-too-weak',
324
+ passed: ['a login session was linked'],
325
+ failed: [],
326
+ },
327
+ ]);
328
+ });
329
+
330
+ it('exits 0 by default, and 1 with --strict when a step is not needed', () => {
331
+ expect(exitCodeOf([report], false)).toBe(0);
332
+ expect(exitCodeOf([report], true)).toBe(1);
333
+ });
334
+
335
+ it('exits 0 with --strict when every step is needed', () => {
336
+ expect(exitCodeOf([{ ...report, steps: report.steps.slice(0, 1) }], true)).toBe(0);
337
+ });
338
+
339
+ it('renders a row per step and a total', () => {
340
+ const table = renderReports([report]);
341
+ expect(table).toContain('unmutated path pass');
342
+ expect(table).toMatch(/^1 +composio login +needed +0\/1$/m);
343
+ expect(table).toContain('failed: a login session was linked');
344
+ expect(table).toMatch(/^3 +composio link slack +check-too-weak +1\/1$/m);
345
+ expect(table).toContain('3 step(s): 1 needed, 1 unused, 1 check-too-weak');
346
+ });
347
+
348
+ describe('when the unmutated path fails a check', () => {
349
+ const broken = reportJourney({
350
+ journey: 'made-up',
351
+ fixture: 'fixtures/made-up.openai.hand-authored.json',
352
+ unmutated: grade(loggedIn, evidenceWith(new World())),
353
+ mutated: [
354
+ { command: 'composio whoami', graded: grade(loggedIn, evidenceWith(linkedWorld())) },
355
+ ],
356
+ });
357
+
358
+ it('reports no step, since no verdict would mean anything', () => {
359
+ expect(broken.unmutated).toEqual({ ok: false, failed: ['a login session was linked'] });
360
+ expect(broken.steps).toEqual([]);
361
+ });
362
+
363
+ it('exits 1 with or without --strict', () => {
364
+ expect(exitCodeOf([report, broken], false)).toBe(1);
365
+ expect(exitCodeOf([broken], true)).toBe(1);
366
+ });
367
+
368
+ it('says so in the table', () => {
369
+ expect(renderReports([broken])).toContain('unmutated path FAILED; no step was dropped');
370
+ expect(renderReports([broken])).toContain('1 unmutated path(s) FAILED');
371
+ });
372
+ });
373
+});