Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | /** |
| 2 | * Runs one journey: a fresh rig, the skill where the agent finds it, the |
| 3 | * journey's starting state, then an agent, and the evidence it left. |
| 4 | * |
| 5 | * The agent is a driver. The replay tests and `record.ts` use the minimal |
| 6 | * loop in `agent.ts`; a live run can also drive Claude Code, see |
| 7 | * `claude-code.ts`. Either way the checks read the same evidence. |
| 8 | */ |
| 9 | |
| 10 | import { startCliRig, type CliRig, type RigOptions } from '@e2e-tests/cli-fake-backend/rig'; |
| 11 | import { worldHash, type JournalEntry } from '@e2e-tests/cli-fake-backend'; |
| 12 | import { runAgent, type ModelAdapter, type Transcript } from './agent'; |
| 13 | import type { Journey, JourneyRun } from './journeys'; |
| 14 | import { BROWSER_PATH, BROWSER_SCRIPT, personFor, type Person } from './person'; |
| 15 | import { scrub, SCRUBBED_BACKEND_URL, SCRUBBED_DASHBOARD_URL } from './scrub'; |
| 16 | import { buildSkill, SKILL_DIRECTORY, systemPrompt, type BuiltSkill } from './skill'; |
| 17 | import type { Usage } from './usage'; |
| 18 | |
| 19 | /** Upper bound on model turns. The longest scripted journey takes eight. */ |
| 20 | export const MAX_TURNS = 12; |
| 21 | |
| 22 | /** Upper bound on one command, so a command that waits for a person cannot stall a run. */ |
| 23 | export const COMMAND_TIMEOUT_SECONDS = 90; |
| 24 | |
| 25 | export interface AdapterInputs { |
| 26 | readonly system: string; |
| 27 | readonly prompt: string; |
| 28 | } |
| 29 | |
| 30 | export interface DriverContext { |
| 31 | readonly rig: CliRig; |
| 32 | readonly journey: Journey; |
| 33 | readonly person: Person; |
| 34 | /** The skill, when the driver places it. */ |
| 35 | readonly skill: BuiltSkill | undefined; |
| 36 | } |
| 37 | |
| 38 | export interface DriverResult { |
| 39 | readonly transcript: Transcript; |
| 40 | /** The requests each command caused, in the order of `transcript.commands`. */ |
| 41 | readonly commandRequests: ReadonlyArray<ReadonlyArray<JournalEntry>>; |
| 42 | readonly usage?: Usage; |
| 43 | } |
| 44 | |
| 45 | export interface Driver { |
| 46 | /** |
| 47 | * Where the skill goes: the system prompt of the minimal loop, the skills |
| 48 | * directory Claude Code reads, or nowhere (the run without the skill). |
| 49 | */ |
| 50 | readonly skill: 'system-prompt' | 'claude-code-skills' | 'none'; |
| 51 | /** |
| 52 | * Whether the agent reads command output with the per-run URLs and clocks |
| 53 | * replaced. Replay needs it; a live agent reads what the CLI printed. |
| 54 | */ |
| 55 | readonly scrubbed: boolean; |
| 56 | readonly readOnlyMounts?: RigOptions['readOnlyMounts']; |
| 57 | drive(context: DriverContext): Promise<DriverResult>; |
| 58 | } |
| 59 | |
| 60 | /** Where Claude Code looks for user skills, under the journey's home. */ |
| 61 | export const claudeSkillDirectory = (home: string): string => `${home}/.claude/skills/composio-cli`; |
| 62 | |
| 63 | const placeSkill = async (rig: CliRig, directory: string, skill: BuiltSkill): Promise<void> => { |
| 64 | await rig.journey.writeFile(`${directory}/SKILL.md`, skill.skillMarkdown); |
| 65 | for (const [name, markdown] of Object.entries(skill.references)) { |
| 66 | await rig.journey.writeFile(`${directory}/references/${name}`, markdown); |
| 67 | } |
| 68 | }; |
| 69 | |
| 70 | export const runJourneyWith = async (journey: Journey, driver: Driver): Promise<JourneyRun> => { |
| 71 | const rig = await startCliRig({ label: journey.id, readOnlyMounts: driver.readOnlyMounts }); |
| 72 | try { |
| 73 | const skill = driver.skill === 'none' ? undefined : buildSkill(); |
| 74 | if (skill && driver.skill === 'system-prompt') await placeSkill(rig, SKILL_DIRECTORY, skill); |
| 75 | if (skill && driver.skill === 'claude-code-skills') { |
| 76 | await placeSkill(rig, claudeSkillDirectory(rig.journey.home), skill); |
| 77 | } |
| 78 | await rig.journey.writeFile(BROWSER_PATH, BROWSER_SCRIPT); |
| 79 | await rig.run(`chmod +x ${BROWSER_PATH}`); |
| 80 | // Nothing is approved by waiting: only the person approves, see `person.ts`. |
| 81 | rig.fake.world.linkCompletesAfterPolls = null; |
| 82 | await journey.setup(rig); |
| 83 | // The person reads what the agent writes, so they see the URLs the agent saw. |
| 84 | const person = personFor( |
| 85 | rig.fake.world, |
| 86 | driver.scrubbed |
| 87 | ? { dashboard: SCRUBBED_DASHBOARD_URL, backend: SCRUBBED_BACKEND_URL } |
| 88 | : { dashboard: rig.fake.docker.https, backend: rig.fake.docker.http } |
| 89 | ); |
| 90 | const startingConnections = rig.fake.world.connections.map(connection => connection.id); |
| 91 | const startingWorldHash = worldHash(rig.fake.world); |
| 92 | |
| 93 | const mark = rig.fake.journal.mark(); |
| 94 | const { transcript, commandRequests, usage } = await driver.drive({ |
| 95 | rig, |
| 96 | journey, |
| 97 | person, |
| 98 | skill, |
| 99 | }); |
| 100 | const requests = rig.fake.journal.since(mark); |
| 101 | const whoami = await rig.run('composio whoami', { timeoutSeconds: COMMAND_TIMEOUT_SECONDS }); |
| 102 | return { |
| 103 | transcript, |
| 104 | rig, |
| 105 | world: rig.fake.world, |
| 106 | requests, |
| 107 | commandRequests, |
| 108 | startingConnections, |
| 109 | startingWorldHash, |
| 110 | whoamiAfter: { exitCode: whoami.exitCode, stdout: whoami.stdout, stderr: whoami.stderr }, |
| 111 | usage, |
| 112 | }; |
| 113 | } catch (error) { |
| 114 | await rig.stop(); |
| 115 | throw error; |
| 116 | } |
| 117 | }; |
| 118 | |
| 119 | /** |
| 120 | * The minimal loop of `agent.ts`. With `skill: 'none'` the model gets no |
| 121 | * system prompt at all. |
| 122 | */ |
| 123 | export const loopDriver = ( |
| 124 | makeAdapter: (inputs: AdapterInputs) => ModelAdapter, |
| 125 | options: { skill?: 'system-prompt' | 'none'; scrubbed?: boolean } = {} |
| 126 | ): Driver => ({ |
| 127 | skill: options.skill ?? 'system-prompt', |
| 128 | scrubbed: options.scrubbed ?? true, |
| 129 | drive: async ({ rig, journey, person, skill }) => { |
| 130 | const commandRequests: Array<ReadonlyArray<JournalEntry>> = []; |
| 131 | const adapter = makeAdapter({ |
| 132 | system: skill ? systemPrompt(skill) : '', |
| 133 | prompt: journey.prompt, |
| 134 | }); |
| 135 | const transcript = await runAgent({ |
| 136 | adapter, |
| 137 | maxTurns: MAX_TURNS, |
| 138 | hear: person.hear, |
| 139 | execute: async command => { |
| 140 | const result = await rig.run(command, { |
| 141 | cwd: journey.cwd, |
| 142 | timeoutSeconds: COMMAND_TIMEOUT_SECONDS, |
| 143 | }); |
| 144 | commandRequests.push(result.requests); |
| 145 | const shown = (text: string) => (options.scrubbed === false ? text : scrub(text, rig.fake)); |
| 146 | return { |
| 147 | exitCode: result.exitCode, |
| 148 | stdout: shown(result.stdout), |
| 149 | stderr: shown(result.stderr), |
| 150 | }; |
| 151 | }, |
| 152 | }); |
| 153 | return { transcript, commandRequests, usage: adapter.usage?.() }; |
| 154 | }, |
| 155 | }); |
| 156 | |
| 157 | export const runJourney = ( |
| 158 | journey: Journey, |
| 159 | makeAdapter: (inputs: AdapterInputs) => ModelAdapter |
| 160 | ): Promise<JourneyRun> => runJourneyWith(journey, loopDriver(makeAdapter)); |