Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Grading a hand-authored path with one command left out, as `mutate.ts` |
| 3 | * does. Kept apart from the script so the verdicts can be tested on made-up |
| 4 | * evidence, without a container. |
| 5 | * |
| 6 | * A dropped command is `needed` when some check fails without it. When none |
| 7 | * fails, the run is compared with the unmutated one by what it left behind: |
| 8 | * the same outcome means the command was `unused`; another outcome means the |
| 9 | * run ended somewhere else and no check noticed, `check-too-weak`. |
| 10 | */ |
| 11 | |
| 12 | import type { Check, Evidence, Journey } from './journeys'; |
| 13 | |
| 14 | export type Verdict = 'needed' | 'unused' | 'check-too-weak'; |
| 15 | |
| 16 | export interface Graded { |
| 17 | readonly checks: ReadonlyArray<Check>; |
| 18 | /** What the run left behind, in a form two runs can be compared by. */ |
| 19 | readonly outcome: string; |
| 20 | } |
| 21 | |
| 22 | export interface StepReport { |
| 23 | /** Position of the dropped command in the path, from 1. */ |
| 24 | readonly step: number; |
| 25 | readonly command: string; |
| 26 | readonly verdict: Verdict; |
| 27 | readonly passed: ReadonlyArray<string>; |
| 28 | readonly failed: ReadonlyArray<string>; |
| 29 | } |
| 30 | |
| 31 | export interface JourneyReport { |
| 32 | readonly journey: string; |
| 33 | readonly fixture: string; |
| 34 | /** The unmutated path. When it fails a check, nothing about the steps can be concluded. */ |
| 35 | readonly unmutated: { readonly ok: boolean; readonly failed: ReadonlyArray<string> }; |
| 36 | readonly steps: ReadonlyArray<StepReport>; |
| 37 | } |
| 38 | |
| 39 | /** |
| 40 | * What the person would find once the agent stopped: the tools that ran, with |
| 41 | * what, as whom, on which project; the connections made during the run; the |
| 42 | * CLI logins; and whether the next command finds a stored login. Reads and |
| 43 | * the order of requests are left out, so a run that only skipped a lookup has |
| 44 | * the same outcome. |
| 45 | */ |
| 46 | export const outcomeOf = (run: Evidence): string => |
| 47 | JSON.stringify({ |
| 48 | executions: run.requests.flatMap(entry => |
| 49 | entry.effect === 'tool-executed' && entry.tool |
| 50 | ? [ |
| 51 | { |
| 52 | tool: entry.tool.slug, |
| 53 | arguments: entry.tool.arguments, |
| 54 | project: entry.project, |
| 55 | user: entry.userId, |
| 56 | }, |
| 57 | ] |
| 58 | : [] |
| 59 | ), |
| 60 | connections: run.world.connections |
| 61 | .filter(connection => !run.startingConnections.includes(connection.id)) |
| 62 | .map(connection => ({ |
| 63 | toolkit: connection.toolkit, |
| 64 | project: connection.projectLabel, |
| 65 | user: connection.userId, |
| 66 | status: connection.status, |
| 67 | })), |
| 68 | logins: run.world.loginSessions.map(session => session.status), |
| 69 | storedLogin: run.whoamiAfter.exitCode === 0, |
| 70 | }); |
| 71 | |
| 72 | export const grade = (journey: Journey, run: Evidence): Graded => ({ |
| 73 | checks: journey.checks(run), |
| 74 | outcome: outcomeOf(run), |
| 75 | }); |
| 76 | |
| 77 | const names = (checks: ReadonlyArray<Check>, ok: boolean) => |
| 78 | checks.filter(check => check.ok === ok).map(check => check.name); |
| 79 | |
| 80 | export const verdictOf = (unmutated: Graded, mutated: Graded): Verdict => { |
| 81 | if (mutated.checks.some(check => !check.ok)) return 'needed'; |
| 82 | return mutated.outcome === unmutated.outcome ? 'unused' : 'check-too-weak'; |
| 83 | }; |
| 84 | |
| 85 | export const reportJourney = (input: { |
| 86 | journey: string; |
| 87 | fixture: string; |
| 88 | unmutated: Graded; |
| 89 | /** One per dropped command, in path order; empty when the unmutated path failed. */ |
| 90 | mutated: ReadonlyArray<{ command: string; graded: Graded }>; |
| 91 | }): JourneyReport => { |
| 92 | const failed = names(input.unmutated.checks, false); |
| 93 | return { |
| 94 | journey: input.journey, |
| 95 | fixture: input.fixture, |
| 96 | unmutated: { ok: failed.length === 0, failed }, |
| 97 | steps: |
| 98 | failed.length > 0 |
| 99 | ? [] |
| 100 | : input.mutated.map(({ command, graded }, index) => ({ |
| 101 | step: index + 1, |
| 102 | command, |
| 103 | verdict: verdictOf(input.unmutated, graded), |
| 104 | passed: names(graded.checks, true), |
| 105 | failed: names(graded.checks, false), |
| 106 | })), |
| 107 | }; |
| 108 | }; |
| 109 | |
| 110 | /** |
| 111 | * Why a run exits non-zero, in the order a reader would fix them. An |
| 112 | * unmutated path that fails means the harness or the CLI is broken, and no |
| 113 | * verdict means anything. With `strict`, a `check-too-weak` step also fails: |
| 114 | * the run ended somewhere else and no check noticed. |
| 115 | * |
| 116 | * `unused` never fails. A read, a `--help`, or a command that fails on purpose |
| 117 | * changes nothing for a check to notice, and the checks grade the outcome, so |
| 118 | * a journey made only of `needed` steps is not a state worth aiming at. |
| 119 | */ |
| 120 | export const failuresOf = ( |
| 121 | reports: ReadonlyArray<JourneyReport>, |
| 122 | strict: boolean |
| 123 | ): ReadonlyArray<string> => [ |
| 124 | ...reports |
| 125 | .filter(report => !report.unmutated.ok) |
| 126 | .map(report => `${report.journey}: the unmutated path failed a check`), |
| 127 | ...(strict |
| 128 | ? reports.flatMap(report => |
| 129 | report.steps |
| 130 | .filter(step => step.verdict === 'check-too-weak') |
| 131 | .map(step => `${report.journey}: step ${step.step} (${step.command}) is check-too-weak`) |
| 132 | ) |
| 133 | : []), |
| 134 | ]; |
| 135 | |
| 136 | /** 1 when `failuresOf` is not empty, else 0. */ |
| 137 | export const exitCodeOf = (reports: ReadonlyArray<JourneyReport>, strict: boolean): number => |
| 138 | failuresOf(reports, strict).length > 0 ? 1 : 0; |
| 139 | |
| 140 | const COMMAND_WIDTH = 60; |
| 141 | |
| 142 | const shorten = (command: string): string => |
| 143 | command.length <= COMMAND_WIDTH ? command : `${command.slice(0, COMMAND_WIDTH - 1)}…`; |
| 144 | |
| 145 | export const renderReports = (reports: ReadonlyArray<JourneyReport>): string => { |
| 146 | const lines: string[] = []; |
| 147 | for (const report of reports) { |
| 148 | lines.push(`mutate: ${report.journey} (${report.fixture})`); |
| 149 | if (!report.unmutated.ok) { |
| 150 | lines.push('unmutated path FAILED; no step was dropped'); |
| 151 | for (const name of report.unmutated.failed) lines.push(` failed: ${name}`); |
| 152 | lines.push(''); |
| 153 | continue; |
| 154 | } |
| 155 | lines.push('unmutated path pass'); |
| 156 | lines.push( |
| 157 | `${'step'.padEnd(5)}${'dropped command'.padEnd(COMMAND_WIDTH + 2)}${'verdict'.padEnd(16)}checks passed` |
| 158 | ); |
| 159 | for (const step of report.steps) { |
| 160 | const total = step.passed.length + step.failed.length; |
| 161 | lines.push( |
| 162 | `${String(step.step).padEnd(5)}${shorten(step.command).padEnd(COMMAND_WIDTH + 2)}${step.verdict.padEnd(16)}${step.passed.length}/${total}` |
| 163 | ); |
| 164 | for (const name of step.failed) lines.push(` failed: ${name}`); |
| 165 | } |
| 166 | lines.push(''); |
| 167 | } |
| 168 | const steps = reports.flatMap(report => report.steps); |
| 169 | const count = (verdict: Verdict) => steps.filter(step => step.verdict === verdict).length; |
| 170 | const broken = reports.filter(report => !report.unmutated.ok).length; |
| 171 | lines.push( |
| 172 | `${steps.length} step(s): ${count('needed')} needed, ${count('unused')} unused, ${count('check-too-weak')} check-too-weak` + |
| 173 | (broken > 0 ? `; ${broken} unmutated path(s) FAILED` : '') |
| 174 | ); |
| 175 | return `${lines.join('\n')}\n`; |
| 176 | }; |