Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Agent journeys, replayed. |
| 3 | * |
| 4 | * Each fixture file is one journey as one model played it. aimock serves that |
| 5 | * file in strict mode, so a request the file does not answer fails the test; |
| 6 | * the journey's checks then look at what the commands did, because a replay |
| 7 | * alone passes even when every command failed. |
| 8 | */ |
| 9 | |
| 10 | import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; |
| 11 | import { LLMock } from '@copilotkit/aimock'; |
| 12 | import { summarize, worldHash } from '@e2e-tests/cli-fake-backend'; |
| 13 | import { startCliRig } from '@e2e-tests/cli-fake-backend/rig'; |
| 14 | import { join } from 'node:path'; |
| 15 | import { adapterFor } from './src/adapters'; |
| 16 | import { HAND_AUTHORED, listFixtures } from './src/fixtures'; |
| 17 | import { JOURNEYS, type JourneyRun } from './src/journeys'; |
| 18 | import { runJourney } from './src/run-journey'; |
| 19 | |
| 20 | const JOURNEY_TIMEOUT_MS = 1_200_000; |
| 21 | |
| 22 | // A fixture's `turnIndex` must hold exactly; aimock otherwise treats it as a hint. |
| 23 | process.env.AIMOCK_STRICT_TURN_INDEX = '1'; |
| 24 | |
| 25 | const fixtures = listFixtures(); |
| 26 | |
| 27 | /** |
| 28 | * How long the idle rig is left alone. Only a request sent on its own within |
| 29 | * this window is seen; a slower timer would get past the guard. |
| 30 | */ |
| 31 | const IDLE_MS = 5_000; |
| 32 | |
| 33 | /** The world each attempt of a journey started from, by journey id. */ |
| 34 | const startingWorlds = new Map<string, Array<{ fixture: string; hash: string }>>(); |
| 35 | |
| 36 | describe('the rig', () => { |
| 37 | // A journey's journal is read as what the agent did. If the binary, or what a |
| 38 | // setup started, kept sending requests once the setup was over, those would |
| 39 | // land in the run's journal and be graded as the agent's. |
| 40 | it( |
| 41 | 'left alone after a login, records no request, changes nothing, and still runs commands', |
| 42 | async () => { |
| 43 | const rig = await startCliRig({ label: 'idle' }); |
| 44 | try { |
| 45 | await rig.login(); |
| 46 | const mark = rig.fake.journal.mark(); |
| 47 | const before = worldHash(rig.fake.world); |
| 48 | await Bun.sleep(IDLE_MS); |
| 49 | expect(summarize(rig.fake.journal.since(mark))).toEqual([]); |
| 50 | expect(worldHash(rig.fake.world)).toBe(before); |
| 51 | const alive = await rig.run('echo alive'); |
| 52 | expect(alive.exitCode).toBe(0); |
| 53 | expect(alive.stdout.trim()).toBe('alive'); |
| 54 | } finally { |
| 55 | await rig.stop(); |
| 56 | } |
| 57 | }, |
| 58 | JOURNEY_TIMEOUT_MS |
| 59 | ); |
| 60 | }); |
| 61 | |
| 62 | describe('the fixture set', () => { |
| 63 | it('has no fixture for a journey that does not exist', () => { |
| 64 | const known = new Set(JOURNEYS.map(journey => journey.id)); |
| 65 | expect(fixtures.filter(fixture => !known.has(fixture.journeyId))).toEqual([]); |
| 66 | }); |
| 67 | |
| 68 | // The baseline: it needs no key to produce, and it is the only thing that |
| 69 | // exercises the OpenAI wire format, so no journey may be without it. |
| 70 | it('has the hand-authored OpenAI path of every journey', () => { |
| 71 | const scripted = new Set( |
| 72 | fixtures |
| 73 | .filter(fixture => fixture.provider === 'openai' && fixture.model === HAND_AUTHORED) |
| 74 | .map(fixture => fixture.journeyId) |
| 75 | ); |
| 76 | expect(JOURNEYS.map(journey => journey.id).filter(id => !scripted.has(id))).toEqual([]); |
| 77 | }); |
| 78 | }); |
| 79 | |
| 80 | for (const journey of JOURNEYS) { |
| 81 | describe(`journey: ${journey.title}`, () => { |
| 82 | const played = fixtures.filter(fixture => fixture.journeyId === journey.id); |
| 83 | |
| 84 | if (!played.some(fixture => fixture.provider === 'anthropic')) { |
| 85 | // A recording is kept only when the run reached the goal; see the README. |
| 86 | it.todo(`anthropic: no recorded run of ${journey.id} reached its goal`); |
| 87 | } |
| 88 | |
| 89 | for (const fixture of played) { |
| 90 | const origin = fixture.model === HAND_AUTHORED ? 'hand-authored' : 'recorded'; |
| 91 | |
| 92 | describe(`${fixture.provider} ${fixture.model} (${origin})`, () => { |
| 93 | let mock: LLMock; |
| 94 | let run: JourneyRun; |
| 95 | |
| 96 | beforeAll(async () => { |
| 97 | mock = new LLMock({ port: 0, strict: true, logLevel: 'silent' }); |
| 98 | mock.loadFixtureFile(fixture.path); |
| 99 | const baseURL = await mock.start(); |
| 100 | run = await runJourney(journey, adapterFor(fixture.provider, fixture.model, baseURL)); |
| 101 | const attempts = startingWorlds.get(journey.id) ?? []; |
| 102 | attempts.push({ |
| 103 | fixture: `${fixture.provider} ${fixture.model}`, |
| 104 | hash: run.startingWorldHash, |
| 105 | }); |
| 106 | startingWorlds.set(journey.id, attempts); |
| 107 | }, JOURNEY_TIMEOUT_MS); |
| 108 | |
| 109 | afterAll(async () => { |
| 110 | // The container and the servers go away even when the log cannot be written. |
| 111 | try { |
| 112 | await run?.rig.journey.writeDebugLog( |
| 113 | join(import.meta.dir, `DEBUG.${journey.id}.${fixture.provider}.${fixture.model}.log`) |
| 114 | ); |
| 115 | } finally { |
| 116 | await run?.rig.stop(); |
| 117 | await mock?.stop(); |
| 118 | } |
| 119 | }); |
| 120 | |
| 121 | it('replays every recorded turn, and nothing else', () => { |
| 122 | const requests = mock.getRequests(); |
| 123 | expect(requests.map(request => request.response.status)).toEqual(requests.map(() => 200)); |
| 124 | // Each fixture answered exactly one request: none skipped, none reused. |
| 125 | const served = requests.map(request => request.response.fixture); |
| 126 | expect(new Set(served).size).toBe(served.length); |
| 127 | expect(served).toHaveLength(mock.getFixtures().length); |
| 128 | }); |
| 129 | |
| 130 | it('ran at least one command, and every command reached the container', () => { |
| 131 | expect(run.transcript.commands.length).toBeGreaterThan(0); |
| 132 | expect(run.rig.journey.steps().map(step => step.command)).toEqual( |
| 133 | expect.arrayContaining(run.transcript.commands.map(command => command.command)) |
| 134 | ); |
| 135 | }); |
| 136 | |
| 137 | it('reached the goal', () => { |
| 138 | const failed = journey.checks(run).filter(check => !check.ok); |
| 139 | expect(failed).toEqual([]); |
| 140 | }); |
| 141 | |
| 142 | if (origin === 'hand-authored') { |
| 143 | // The scripted path is the one the skill teaches; it never leaves its tree. |
| 144 | it('took no detour', () => { |
| 145 | const taken = (journey.detours?.(run) ?? []).filter(check => !check.ok); |
| 146 | expect(taken).toEqual([]); |
| 147 | }); |
| 148 | } |
| 149 | |
| 150 | if (origin === 'hand-authored' && journey.scriptedChecks) { |
| 151 | const { scriptedChecks } = journey; |
| 152 | it('the CLI answered each scripted command as described', () => { |
| 153 | const failed = scriptedChecks(run).filter(check => !check.ok); |
| 154 | expect(failed).toEqual([]); |
| 155 | }); |
| 156 | } |
| 157 | }); |
| 158 | } |
| 159 | }); |
| 160 | } |
| 161 | |
| 162 | // Runs after every journey above: bun runs a file's tests in the order they are |
| 163 | // defined. pass^k compares attempts of one journey, which says nothing about |
| 164 | // the agent if the fake handed them different starting worlds. |
| 165 | describe('attempt isolation', () => { |
| 166 | it('started every attempt of a journey from the same world', () => { |
| 167 | // An empty map would pass the comparison below without comparing anything. |
| 168 | expect(startingWorlds.size).toBeGreaterThan(0); |
| 169 | const differing = [...startingWorlds].filter( |
| 170 | ([, attempts]) => new Set(attempts.map(attempt => attempt.hash)).size > 1 |
| 171 | ); |
| 172 | expect(Object.fromEntries(differing)).toEqual({}); |
| 173 | }); |
| 174 | }); |