Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | /** |
| 2 | * Grader mutation: plays each journey's hand-authored path with one command |
| 3 | * left out, and reports whether the journey's checks noticed. |
| 4 | * |
| 5 | * bun run mutate.ts [--journeys <id,id>] [--json] [--strict] |
| 6 | * |
| 7 | * The path is played with no model (`src/scripted.ts`) against a fresh rig, |
| 8 | * first whole, then once per dropped command, and graded by the same checks as |
| 9 | * a replay. Verdicts are explained in `src/mutation.ts`. |
| 10 | * |
| 11 | * What a report shows: which commands of the taught path the checks depend |
| 12 | * on, and which could go without any check failing. What it does not show: |
| 13 | * that the checks catch a wrong path the hand-authored file never takes, or |
| 14 | * anything about a real model. |
| 15 | * |
| 16 | * Exits 1 when an unmutated path fails a check, and with `--strict` also when |
| 17 | * a step is `check-too-weak`: a way to end somewhere else that no check |
| 18 | * noticed. `unused` is reported and never fails, because a read or a command |
| 19 | * that fails on purpose changes nothing for an outcome check to notice. |
| 20 | * Results go to `results/<batch>/` (git-ignored). Needs Docker; spends nothing. |
| 21 | */ |
| 22 | |
| 23 | import { mkdirSync, writeFileSync } from 'node:fs'; |
| 24 | import { join, relative } from 'node:path'; |
| 25 | import { parseArgs } from 'node:util'; |
| 26 | import { HAND_AUTHORED, listFixtures } from './src/fixtures'; |
| 27 | import { journeyById, JOURNEYS, type Journey } from './src/journeys'; |
| 28 | import { |
| 29 | exitCodeOf, |
| 30 | failuresOf, |
| 31 | grade, |
| 32 | renderReports, |
| 33 | reportJourney, |
| 34 | type Graded, |
| 35 | type JourneyReport, |
| 36 | } from './src/mutation'; |
| 37 | import { runJourneyWith } from './src/run-journey'; |
| 38 | import { readScriptedPath, scriptedCommands, scriptedDriver } from './src/scripted'; |
| 39 | |
| 40 | const HERE = import.meta.dir; |
| 41 | |
| 42 | const { values } = parseArgs({ |
| 43 | args: process.argv.slice(2), |
| 44 | options: { |
| 45 | journeys: { type: 'string' }, |
| 46 | json: { type: 'boolean', default: false }, |
| 47 | strict: { type: 'boolean', default: false }, |
| 48 | }, |
| 49 | }); |
| 50 | |
| 51 | const selected = (): ReadonlyArray<Journey> => { |
| 52 | try { |
| 53 | return values.journeys ? values.journeys.split(',').map(journeyById) : JOURNEYS; |
| 54 | } catch (error) { |
| 55 | console.error(error instanceof Error ? error.message : String(error)); |
| 56 | console.error('Usage: bun run mutate.ts [--journeys <id,id>] [--json] [--strict]'); |
| 57 | process.exit(2); |
| 58 | } |
| 59 | }; |
| 60 | const journeys = selected(); |
| 61 | |
| 62 | const scripted = new Map( |
| 63 | listFixtures() |
| 64 | .filter(fixture => fixture.provider === 'openai' && fixture.model === HAND_AUTHORED) |
| 65 | .map(fixture => [fixture.journeyId, fixture.path]) |
| 66 | ); |
| 67 | |
| 68 | const play = async (journey: Journey, path: string, drop?: number): Promise<Graded> => { |
| 69 | const run = await runJourneyWith(journey, scriptedDriver(path, { drop })); |
| 70 | try { |
| 71 | return grade(journey, run); |
| 72 | } finally { |
| 73 | await run.rig.stop(); |
| 74 | } |
| 75 | }; |
| 76 | |
| 77 | const reports: JourneyReport[] = []; |
| 78 | for (const journey of journeys) { |
| 79 | const path = scripted.get(journey.id); |
| 80 | // e2e.test.ts already fails when a journey has no hand-authored path. |
| 81 | if (!path) throw new Error(`${journey.id} has no hand-authored fixture`); |
| 82 | const fixture = relative(HERE, path); |
| 83 | const commands = scriptedCommands(readScriptedPath(path)); |
| 84 | console.error(`${journey.id}: unmutated path`); |
| 85 | const unmutated = await play(journey, path); |
| 86 | const mutated: Array<{ command: string; graded: Graded }> = []; |
| 87 | if (unmutated.checks.every(check => check.ok)) { |
| 88 | for (const [index, command] of commands.entries()) { |
| 89 | console.error(`${journey.id}: without step ${index + 1}, ${command.slice(0, 80)}`); |
| 90 | mutated.push({ command, graded: await play(journey, path, index) }); |
| 91 | } |
| 92 | } |
| 93 | reports.push(reportJourney({ journey: journey.id, fixture, unmutated, mutated })); |
| 94 | } |
| 95 | |
| 96 | const table = renderReports(reports); |
| 97 | const batch = `${new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)}-mutate`; |
| 98 | const directory = join(HERE, 'results', batch); |
| 99 | mkdirSync(directory, { recursive: true }); |
| 100 | writeFileSync(join(directory, 'mutations.json'), `${JSON.stringify(reports, null, 2)}\n`); |
| 101 | writeFileSync(join(directory, 'summary.txt'), table); |
| 102 | |
| 103 | process.stdout.write(values.json ? `${JSON.stringify(reports, null, 2)}\n` : table); |
| 104 | console.error(`written ${relative(process.cwd(), directory)}`); |
| 105 | const failures = failuresOf(reports, values.strict ?? false); |
| 106 | for (const failure of failures) console.error(`failed ${failure}`); |
| 107 | process.exit(exitCodeOf(reports, values.strict ?? false)); |