Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
ts 107 lines 4.21 KB
Raw
1 /**
2 * Grader mutation: plays each journey's hand-authored path with one command
3 * left out, and reports whether the journey's checks noticed.
4 *
5 * bun run mutate.ts [--journeys <id,id>] [--json] [--strict]
6 *
7 * The path is played with no model (`src/scripted.ts`) against a fresh rig,
8 * first whole, then once per dropped command, and graded by the same checks as
9 * a replay. Verdicts are explained in `src/mutation.ts`.
10 *
11 * What a report shows: which commands of the taught path the checks depend
12 * on, and which could go without any check failing. What it does not show:
13 * that the checks catch a wrong path the hand-authored file never takes, or
14 * anything about a real model.
15 *
16 * Exits 1 when an unmutated path fails a check, and with `--strict` also when
17 * a step is `check-too-weak`: a way to end somewhere else that no check
18 * noticed. `unused` is reported and never fails, because a read or a command
19 * that fails on purpose changes nothing for an outcome check to notice.
20 * Results go to `results/<batch>/` (git-ignored). Needs Docker; spends nothing.
21 */
22
23 import { mkdirSync, writeFileSync } from 'node:fs';
24 import { join, relative } from 'node:path';
25 import { parseArgs } from 'node:util';
26 import { HAND_AUTHORED, listFixtures } from './src/fixtures';
27 import { journeyById, JOURNEYS, type Journey } from './src/journeys';
28 import {
29 exitCodeOf,
30 failuresOf,
31 grade,
32 renderReports,
33 reportJourney,
34 type Graded,
35 type JourneyReport,
36 } from './src/mutation';
37 import { runJourneyWith } from './src/run-journey';
38 import { readScriptedPath, scriptedCommands, scriptedDriver } from './src/scripted';
39
40 const HERE = import.meta.dir;
41
42 const { values } = parseArgs({
43 args: process.argv.slice(2),
44 options: {
45 journeys: { type: 'string' },
46 json: { type: 'boolean', default: false },
47 strict: { type: 'boolean', default: false },
48 },
49 });
50
51 const selected = (): ReadonlyArray<Journey> => {
52 try {
53 return values.journeys ? values.journeys.split(',').map(journeyById) : JOURNEYS;
54 } catch (error) {
55 console.error(error instanceof Error ? error.message : String(error));
56 console.error('Usage: bun run mutate.ts [--journeys <id,id>] [--json] [--strict]');
57 process.exit(2);
58 }
59 };
60 const journeys = selected();
61
62 const scripted = new Map(
63 listFixtures()
64 .filter(fixture => fixture.provider === 'openai' && fixture.model === HAND_AUTHORED)
65 .map(fixture => [fixture.journeyId, fixture.path])
66 );
67
68 const play = async (journey: Journey, path: string, drop?: number): Promise<Graded> => {
69 const run = await runJourneyWith(journey, scriptedDriver(path, { drop }));
70 try {
71 return grade(journey, run);
72 } finally {
73 await run.rig.stop();
74 }
75 };
76
77 const reports: JourneyReport[] = [];
78 for (const journey of journeys) {
79 const path = scripted.get(journey.id);
80 // e2e.test.ts already fails when a journey has no hand-authored path.
81 if (!path) throw new Error(`${journey.id} has no hand-authored fixture`);
82 const fixture = relative(HERE, path);
83 const commands = scriptedCommands(readScriptedPath(path));
84 console.error(`${journey.id}: unmutated path`);
85 const unmutated = await play(journey, path);
86 const mutated: Array<{ command: string; graded: Graded }> = [];
87 if (unmutated.checks.every(check => check.ok)) {
88 for (const [index, command] of commands.entries()) {
89 console.error(`${journey.id}: without step ${index + 1}, ${command.slice(0, 80)}`);
90 mutated.push({ command, graded: await play(journey, path, index) });
91 }
92 }
93 reports.push(reportJourney({ journey: journey.id, fixture, unmutated, mutated }));
94 }
95
96 const table = renderReports(reports);
97 const batch = `${new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)}-mutate`;
98 const directory = join(HERE, 'results', batch);
99 mkdirSync(directory, { recursive: true });
100 writeFileSync(join(directory, 'mutations.json'), `${JSON.stringify(reports, null, 2)}\n`);
101 writeFileSync(join(directory, 'summary.txt'), table);
102
103 process.stdout.write(values.json ? `${JSON.stringify(reports, null, 2)}\n` : table);
104 console.error(`written ${relative(process.cwd(), directory)}`);
105 const failures = failuresOf(reports, values.strict ?? false);
106 for (const failure of failures) console.error(`failed ${failure}`);
107 process.exit(exitCodeOf(reports, values.strict ?? false));