@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 04:45 UTC

next
ts 176 lines 6.49 KB
Raw
1 /**
2 * Grading a hand-authored path with one command left out, as `mutate.ts`
3 * does. Kept apart from the script so the verdicts can be tested on made-up
4 * evidence, without a container.
5 *
6 * A dropped command is `needed` when some check fails without it. When none
7 * fails, the run is compared with the unmutated one by what it left behind:
8 * the same outcome means the command was `unused`; another outcome means the
9 * run ended somewhere else and no check noticed, `check-too-weak`.
10 */
11
12 import type { Check, Evidence, Journey } from './journeys';
13
14 export type Verdict = 'needed' | 'unused' | 'check-too-weak';
15
16 export interface Graded {
17 readonly checks: ReadonlyArray<Check>;
18 /** What the run left behind, in a form two runs can be compared by. */
19 readonly outcome: string;
20 }
21
22 export interface StepReport {
23 /** Position of the dropped command in the path, from 1. */
24 readonly step: number;
25 readonly command: string;
26 readonly verdict: Verdict;
27 readonly passed: ReadonlyArray<string>;
28 readonly failed: ReadonlyArray<string>;
29 }
30
31 export interface JourneyReport {
32 readonly journey: string;
33 readonly fixture: string;
34 /** The unmutated path. When it fails a check, nothing about the steps can be concluded. */
35 readonly unmutated: { readonly ok: boolean; readonly failed: ReadonlyArray<string> };
36 readonly steps: ReadonlyArray<StepReport>;
37 }
38
39 /**
40 * What the person would find once the agent stopped: the tools that ran, with
41 * what, as whom, on which project; the connections made during the run; the
42 * CLI logins; and whether the next command finds a stored login. Reads and
43 * the order of requests are left out, so a run that only skipped a lookup has
44 * the same outcome.
45 */
46 export const outcomeOf = (run: Evidence): string =>
47 JSON.stringify({
48 executions: run.requests.flatMap(entry =>
49 entry.effect === 'tool-executed' && entry.tool
50 ? [
51 {
52 tool: entry.tool.slug,
53 arguments: entry.tool.arguments,
54 project: entry.project,
55 user: entry.userId,
56 },
57 ]
58 : []
59 ),
60 connections: run.world.connections
61 .filter(connection => !run.startingConnections.includes(connection.id))
62 .map(connection => ({
63 toolkit: connection.toolkit,
64 project: connection.projectLabel,
65 user: connection.userId,
66 status: connection.status,
67 })),
68 logins: run.world.loginSessions.map(session => session.status),
69 storedLogin: run.whoamiAfter.exitCode === 0,
70 });
71
72 export const grade = (journey: Journey, run: Evidence): Graded => ({
73 checks: journey.checks(run),
74 outcome: outcomeOf(run),
75 });
76
77 const names = (checks: ReadonlyArray<Check>, ok: boolean) =>
78 checks.filter(check => check.ok === ok).map(check => check.name);
79
80 export const verdictOf = (unmutated: Graded, mutated: Graded): Verdict => {
81 if (mutated.checks.some(check => !check.ok)) return 'needed';
82 return mutated.outcome === unmutated.outcome ? 'unused' : 'check-too-weak';
83 };
84
85 export const reportJourney = (input: {
86 journey: string;
87 fixture: string;
88 unmutated: Graded;
89 /** One per dropped command, in path order; empty when the unmutated path failed. */
90 mutated: ReadonlyArray<{ command: string; graded: Graded }>;
91 }): JourneyReport => {
92 const failed = names(input.unmutated.checks, false);
93 return {
94 journey: input.journey,
95 fixture: input.fixture,
96 unmutated: { ok: failed.length === 0, failed },
97 steps:
98 failed.length > 0
99 ? []
100 : input.mutated.map(({ command, graded }, index) => ({
101 step: index + 1,
102 command,
103 verdict: verdictOf(input.unmutated, graded),
104 passed: names(graded.checks, true),
105 failed: names(graded.checks, false),
106 })),
107 };
108 };
109
110 /**
111 * Why a run exits non-zero, in the order a reader would fix them. An
112 * unmutated path that fails means the harness or the CLI is broken, and no
113 * verdict means anything. With `strict`, a `check-too-weak` step also fails:
114 * the run ended somewhere else and no check noticed.
115 *
116 * `unused` never fails. A read, a `--help`, or a command that fails on purpose
117 * changes nothing for a check to notice, and the checks grade the outcome, so
118 * a journey made only of `needed` steps is not a state worth aiming at.
119 */
120 export const failuresOf = (
121 reports: ReadonlyArray<JourneyReport>,
122 strict: boolean
123 ): ReadonlyArray<string> => [
124 ...reports
125 .filter(report => !report.unmutated.ok)
126 .map(report => `${report.journey}: the unmutated path failed a check`),
127 ...(strict
128 ? reports.flatMap(report =>
129 report.steps
130 .filter(step => step.verdict === 'check-too-weak')
131 .map(step => `${report.journey}: step ${step.step} (${step.command}) is check-too-weak`)
132 )
133 : []),
134 ];
135
136 /** 1 when `failuresOf` is not empty, else 0. */
137 export const exitCodeOf = (reports: ReadonlyArray<JourneyReport>, strict: boolean): number =>
138 failuresOf(reports, strict).length > 0 ? 1 : 0;
139
140 const COMMAND_WIDTH = 60;
141
142 const shorten = (command: string): string =>
143 command.length <= COMMAND_WIDTH ? command : `${command.slice(0, COMMAND_WIDTH - 1)}…`;
144
145 export const renderReports = (reports: ReadonlyArray<JourneyReport>): string => {
146 const lines: string[] = [];
147 for (const report of reports) {
148 lines.push(`mutate: ${report.journey} (${report.fixture})`);
149 if (!report.unmutated.ok) {
150 lines.push('unmutated path FAILED; no step was dropped');
151 for (const name of report.unmutated.failed) lines.push(` failed: ${name}`);
152 lines.push('');
153 continue;
154 }
155 lines.push('unmutated path pass');
156 lines.push(
157 `${'step'.padEnd(5)}${'dropped command'.padEnd(COMMAND_WIDTH + 2)}${'verdict'.padEnd(16)}checks passed`
158 );
159 for (const step of report.steps) {
160 const total = step.passed.length + step.failed.length;
161 lines.push(
162 `${String(step.step).padEnd(5)}${shorten(step.command).padEnd(COMMAND_WIDTH + 2)}${step.verdict.padEnd(16)}${step.passed.length}/${total}`
163 );
164 for (const name of step.failed) lines.push(` failed: ${name}`);
165 }
166 lines.push('');
167 }
168 const steps = reports.flatMap(report => report.steps);
169 const count = (verdict: Verdict) => steps.filter(step => step.verdict === verdict).length;
170 const broken = reports.filter(report => !report.unmutated.ok).length;
171 lines.push(
172 `${steps.length} step(s): ${count('needed')} needed, ${count('unused')} unused, ${count('check-too-weak')} check-too-weak` +
173 (broken > 0 ? `; ${broken} unmutated path(s) FAILED` : '')
174 );
175 return `${lines.join('\n')}\n`;
176 };