Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 174 lines 6.92 KB
Raw
1 /**
2 * Agent journeys, replayed.
3 *
4 * Each fixture file is one journey as one model played it. aimock serves that
5 * file in strict mode, so a request the file does not answer fails the test;
6 * the journey's checks then look at what the commands did, because a replay
7 * alone passes even when every command failed.
8 */
9
10 import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
11 import { LLMock } from '@copilotkit/aimock';
12 import { summarize, worldHash } from '@e2e-tests/cli-fake-backend';
13 import { startCliRig } from '@e2e-tests/cli-fake-backend/rig';
14 import { join } from 'node:path';
15 import { adapterFor } from './src/adapters';
16 import { HAND_AUTHORED, listFixtures } from './src/fixtures';
17 import { JOURNEYS, type JourneyRun } from './src/journeys';
18 import { runJourney } from './src/run-journey';
19
20 const JOURNEY_TIMEOUT_MS = 1_200_000;
21
22 // A fixture's `turnIndex` must hold exactly; aimock otherwise treats it as a hint.
23 process.env.AIMOCK_STRICT_TURN_INDEX = '1';
24
25 const fixtures = listFixtures();
26
27 /**
28 * How long the idle rig is left alone. Only a request sent on its own within
29 * this window is seen; a slower timer would get past the guard.
30 */
31 const IDLE_MS = 5_000;
32
33 /** The world each attempt of a journey started from, by journey id. */
34 const startingWorlds = new Map<string, Array<{ fixture: string; hash: string }>>();
35
36 describe('the rig', () => {
37 // A journey's journal is read as what the agent did. If the binary, or what a
38 // setup started, kept sending requests once the setup was over, those would
39 // land in the run's journal and be graded as the agent's.
40 it(
41 'left alone after a login, records no request, changes nothing, and still runs commands',
42 async () => {
43 const rig = await startCliRig({ label: 'idle' });
44 try {
45 await rig.login();
46 const mark = rig.fake.journal.mark();
47 const before = worldHash(rig.fake.world);
48 await Bun.sleep(IDLE_MS);
49 expect(summarize(rig.fake.journal.since(mark))).toEqual([]);
50 expect(worldHash(rig.fake.world)).toBe(before);
51 const alive = await rig.run('echo alive');
52 expect(alive.exitCode).toBe(0);
53 expect(alive.stdout.trim()).toBe('alive');
54 } finally {
55 await rig.stop();
56 }
57 },
58 JOURNEY_TIMEOUT_MS
59 );
60 });
61
62 describe('the fixture set', () => {
63 it('has no fixture for a journey that does not exist', () => {
64 const known = new Set(JOURNEYS.map(journey => journey.id));
65 expect(fixtures.filter(fixture => !known.has(fixture.journeyId))).toEqual([]);
66 });
67
68 // The baseline: it needs no key to produce, and it is the only thing that
69 // exercises the OpenAI wire format, so no journey may be without it.
70 it('has the hand-authored OpenAI path of every journey', () => {
71 const scripted = new Set(
72 fixtures
73 .filter(fixture => fixture.provider === 'openai' && fixture.model === HAND_AUTHORED)
74 .map(fixture => fixture.journeyId)
75 );
76 expect(JOURNEYS.map(journey => journey.id).filter(id => !scripted.has(id))).toEqual([]);
77 });
78 });
79
80 for (const journey of JOURNEYS) {
81 describe(`journey: ${journey.title}`, () => {
82 const played = fixtures.filter(fixture => fixture.journeyId === journey.id);
83
84 if (!played.some(fixture => fixture.provider === 'anthropic')) {
85 // A recording is kept only when the run reached the goal; see the README.
86 it.todo(`anthropic: no recorded run of ${journey.id} reached its goal`);
87 }
88
89 for (const fixture of played) {
90 const origin = fixture.model === HAND_AUTHORED ? 'hand-authored' : 'recorded';
91
92 describe(`${fixture.provider} ${fixture.model} (${origin})`, () => {
93 let mock: LLMock;
94 let run: JourneyRun;
95
96 beforeAll(async () => {
97 mock = new LLMock({ port: 0, strict: true, logLevel: 'silent' });
98 mock.loadFixtureFile(fixture.path);
99 const baseURL = await mock.start();
100 run = await runJourney(journey, adapterFor(fixture.provider, fixture.model, baseURL));
101 const attempts = startingWorlds.get(journey.id) ?? [];
102 attempts.push({
103 fixture: `${fixture.provider} ${fixture.model}`,
104 hash: run.startingWorldHash,
105 });
106 startingWorlds.set(journey.id, attempts);
107 }, JOURNEY_TIMEOUT_MS);
108
109 afterAll(async () => {
110 // The container and the servers go away even when the log cannot be written.
111 try {
112 await run?.rig.journey.writeDebugLog(
113 join(import.meta.dir, `DEBUG.${journey.id}.${fixture.provider}.${fixture.model}.log`)
114 );
115 } finally {
116 await run?.rig.stop();
117 await mock?.stop();
118 }
119 });
120
121 it('replays every recorded turn, and nothing else', () => {
122 const requests = mock.getRequests();
123 expect(requests.map(request => request.response.status)).toEqual(requests.map(() => 200));
124 // Each fixture answered exactly one request: none skipped, none reused.
125 const served = requests.map(request => request.response.fixture);
126 expect(new Set(served).size).toBe(served.length);
127 expect(served).toHaveLength(mock.getFixtures().length);
128 });
129
130 it('ran at least one command, and every command reached the container', () => {
131 expect(run.transcript.commands.length).toBeGreaterThan(0);
132 expect(run.rig.journey.steps().map(step => step.command)).toEqual(
133 expect.arrayContaining(run.transcript.commands.map(command => command.command))
134 );
135 });
136
137 it('reached the goal', () => {
138 const failed = journey.checks(run).filter(check => !check.ok);
139 expect(failed).toEqual([]);
140 });
141
142 if (origin === 'hand-authored') {
143 // The scripted path is the one the skill teaches; it never leaves its tree.
144 it('took no detour', () => {
145 const taken = (journey.detours?.(run) ?? []).filter(check => !check.ok);
146 expect(taken).toEqual([]);
147 });
148 }
149
150 if (origin === 'hand-authored' && journey.scriptedChecks) {
151 const { scriptedChecks } = journey;
152 it('the CLI answered each scripted command as described', () => {
153 const failed = scriptedChecks(run).filter(check => !check.ok);
154 expect(failed).toEqual([]);
155 });
156 }
157 });
158 }
159 });
160 }
161
162 // Runs after every journey above: bun runs a file's tests in the order they are
163 // defined. pass^k compares attempts of one journey, which says nothing about
164 // the agent if the fake handed them different starting worlds.
165 describe('attempt isolation', () => {
166 it('started every attempt of a journey from the same world', () => {
167 // An empty map would pass the comparison below without comparing anything.
168 expect(startingWorlds.size).toBeGreaterThan(0);
169 const differing = [...startingWorlds].filter(
170 ([, attempts]) => new Set(attempts.map(attempt => attempt.hash)).size > 1
171 );
172 expect(Object.fromEntries(differing)).toEqual({});
173 });
174 });