The best way to control Claude Code, Codex and more from any computer or phone
| 1 | #!/usr/bin/env node |
| 2 | // @effect-diagnostics nodeBuiltinImport:off globalConsole:off |
| 3 | /** |
| 4 | * `pnpm mcp-tool-use:compare <results.jsonl>... [--model <id>]`: per case and arm, pass rate and |
| 5 | * forge spend. Rows are regraded with this checkout's cases. Cache writes are priced as reads, so |
| 6 | * run order does not pick the cheaper arm; only Claude agents are priced. |
| 7 | */ |
| 8 | import * as NodeFS from "node:fs"; |
| 9 | import * as NodeUtil from "node:util"; |
| 10 | |
| 11 | import * as Schema from "effect/Schema"; |
| 12 | |
| 13 | import { |
| 14 | gradeSample, |
| 15 | McpToolUseCase, |
| 16 | modelRequests, |
| 17 | pastedText, |
| 18 | PRICES, |
| 19 | worldFilePaths, |
| 20 | } from "./run.ts"; |
| 21 | |
| 22 | const { values, positionals } = NodeUtil.parseArgs({ |
| 23 | allowPositionals: true, |
| 24 | options: { |
| 25 | model: { type: "string", default: "claude-sonnet-5-5" }, |
| 26 | // Rows from before shellModel was recorded: the shell model they ran with (the runner's default). |
| 27 | "shell-model": { type: "string", default: "claude-haiku-5-5" }, |
| 28 | cases: { |
| 29 | type: "string", |
| 30 | default: new URL("./cases/pr-tools-vs-cli.ndjson", import.meta.url).pathname, |
| 31 | }, |
| 32 | }, |
| 33 | }); |
| 34 | const decodeCase = Schema.decodeUnknownSync(Schema.fromJsonString(McpToolUseCase)); |
| 35 | const cases = new Map( |
| 36 | NodeFS.readFileSync(values.cases, "utf8") |
| 37 | .split("\n") |
| 38 | .filter((line) => line.trim() !== "") |
| 39 | .map((line) => decodeCase(line)) |
| 40 | .map((testCase) => [testCase.id, testCase]), |
| 41 | ); |
| 42 | |
| 43 | type Usage = { input: number; output: number; cacheWrite: number; cacheRead: number }; |
| 44 | type Call = { step: number; tool: string; input: unknown; failed?: boolean; output?: string }; |
| 45 | type Row = { |
| 46 | id: string; |
| 47 | arm?: string; |
| 48 | pass: boolean; |
| 49 | error?: string; |
| 50 | stopReason?: string; |
| 51 | calls?: ReadonlyArray<Call>; |
| 52 | modelCalls?: number; |
| 53 | shellModel?: string; |
| 54 | usage: Usage; |
| 55 | shellUsage: Usage; |
| 56 | }; |
| 57 | |
| 58 | const FORGE_TOOL = |
| 59 | /^(pull_request_\w+|ci_logs|ci_rerun|(un)?watch_pull_request|(un)?link_pull_requests?|list_thread_pull_requests)$/; |
| 60 | const FORGE_COMMAND = /(^|[\s;&|(`$])(gh|cafe|glab|tea|src)\s|api\.github\.com|git\.cafe\/api/; |
| 61 | const command = (call: Call) => |
| 62 | call.tool === "Bash" && typeof call.input === "object" && call.input !== null |
| 63 | ? String((call.input as { command?: unknown }).command ?? "") |
| 64 | : ""; |
| 65 | const isForge = (call: Call) => FORGE_TOOL.test(call.tool) || FORGE_COMMAND.test(command(call)); |
| 66 | const sleepSeconds = (call: Call) => |
| 67 | [...command(call).matchAll(/\bsleep\s+(\d+)/g)].reduce((sum, [, s]) => sum + Number(s), 0) + |
| 68 | (call.tool === "sleep" || call.tool === "Monitor" ? 60 : 0); |
| 69 | |
| 70 | const price = (usage: Usage, model: string) => { |
| 71 | const rates = (PRICES as Record<string, Usage | undefined>)[model]; |
| 72 | if (rates === undefined) return Number.NaN; |
| 73 | return ( |
| 74 | (usage.input * rates.input + |
| 75 | usage.output * rates.output + |
| 76 | usage.cacheWrite * rates.cacheRead + |
| 77 | usage.cacheRead * rates.cacheRead) / |
| 78 | 1e6 |
| 79 | ); |
| 80 | }; |
| 81 | |
| 82 | const graded = positionals.flatMap((file) => |
| 83 | NodeFS.readFileSync(file, "utf8") |
| 84 | .split("\n") |
| 85 | .filter((line) => line.trim() !== "") |
| 86 | .map((line) => JSON.parse(line) as Row) |
| 87 | .filter((row) => row.error === undefined) |
| 88 | .map((row) => { |
| 89 | const testCase = cases.get(row.id); |
| 90 | if (testCase === undefined) return undefined; |
| 91 | const expect = row.arm === "raw" ? (testCase.expectRaw ?? testCase.expect) : testCase.expect; |
| 92 | const checks = gradeSample( |
| 93 | expect, |
| 94 | row as unknown as Parameters<typeof gradeSample>[1], |
| 95 | pastedText(testCase), |
| 96 | worldFilePaths(testCase), |
| 97 | ); |
| 98 | return { ...row, pass: checks.every((check) => check.pass) }; |
| 99 | }), |
| 100 | ); |
| 101 | // A row whose case is not in --cases cannot be regraded, so it stays out of every number. |
| 102 | const rows: Array<Row> = graded.filter((row) => row !== undefined); |
| 103 | const unmatched = graded.length - rows.length; |
| 104 | |
| 105 | type Stats = { |
| 106 | n: number; |
| 107 | pass: number; |
| 108 | steps: number; |
| 109 | forge: number; |
| 110 | failed: number; |
| 111 | bytes: number; |
| 112 | sleep: number; |
| 113 | help: number; |
| 114 | tokens: number; |
| 115 | cost: number; |
| 116 | }; |
| 117 | const empty = (): Stats => ({ |
| 118 | n: 0, |
| 119 | pass: 0, |
| 120 | steps: 0, |
| 121 | forge: 0, |
| 122 | failed: 0, |
| 123 | bytes: 0, |
| 124 | sleep: 0, |
| 125 | help: 0, |
| 126 | tokens: 0, |
| 127 | cost: 0, |
| 128 | }); |
| 129 | const add = (stats: Stats, row: Row) => { |
| 130 | const calls = row.calls ?? []; |
| 131 | const forge = calls.filter(isForge); |
| 132 | stats.n++; |
| 133 | if (row.pass) stats.pass++; |
| 134 | stats.steps += modelRequests(row); |
| 135 | stats.forge += forge.length; |
| 136 | stats.failed += forge.filter((call) => call.failed === true).length; |
| 137 | stats.bytes += forge.reduce((sum, call) => sum + (call.output?.length ?? 0), 0); |
| 138 | stats.sleep += calls.reduce((sum, call) => sum + sleepSeconds(call), 0); |
| 139 | stats.help += forge.filter((call) => /--help\b|\shelp\s*$/.test(command(call))).length; |
| 140 | stats.tokens += row.usage.input + row.usage.cacheWrite + row.usage.cacheRead + row.usage.output; |
| 141 | stats.cost += |
| 142 | price(row.usage, values.model) + price(row.shellUsage, row.shellModel ?? values["shell-model"]); |
| 143 | }; |
| 144 | |
| 145 | const byCase = new Map<string, Map<string, Stats>>(); |
| 146 | const totals = new Map<string, Stats>(); |
| 147 | for (const row of rows) { |
| 148 | const arm = row.arm ?? "tools"; |
| 149 | const perArm = byCase.get(row.id) ?? new Map<string, Stats>(); |
| 150 | byCase.set(row.id, perArm); |
| 151 | const stats = perArm.get(arm) ?? empty(); |
| 152 | perArm.set(arm, stats); |
| 153 | add(stats, row); |
| 154 | const total = totals.get(arm) ?? empty(); |
| 155 | totals.set(arm, total); |
| 156 | add(total, row); |
| 157 | } |
| 158 | |
| 159 | const mean = (value: number, n: number, digits = 1) => |
| 160 | n === 0 ? "-" : (value / n).toFixed(digits); |
| 161 | const cells = (s: Stats) => |
| 162 | [ |
| 163 | `${s.pass}/${s.n}`, |
| 164 | mean(s.steps, s.n), |
| 165 | mean(s.forge, s.n), |
| 166 | mean(s.failed, s.n), |
| 167 | mean(s.bytes / 1024, s.n), |
| 168 | mean(s.sleep, s.n, 0), |
| 169 | mean(s.help, s.n), |
| 170 | mean(s.tokens / 1000, s.n, 0), |
| 171 | Number.isNaN(s.cost) ? "-" : `$${mean(s.cost, s.n, 3)}`, |
| 172 | ].join(" | "); |
| 173 | const header = |
| 174 | "| case | arm | pass | model calls | forge calls | failed | forge KB | sleep s | --help | ktokens | $/sample |"; |
| 175 | console.log( |
| 176 | `# Tools vs raw CLI (${values.model})\n\nMeans per sample.${unmatched > 0 ? ` Left out ${unmatched} rows whose case is not in ${values.cases}.` : ""}\n\n${header}`, |
| 177 | ); |
| 178 | console.log(`| --- | --- | ${Array.from({ length: 9 }, () => "---").join(" | ")} |`); |
| 179 | for (const [id, perArm] of [...byCase].toSorted(([a], [b]) => a.localeCompare(b))) { |
| 180 | for (const [arm, stats] of [...perArm].toSorted(([a], [b]) => b.localeCompare(a))) |
| 181 | console.log(`| ${id} | ${arm} | ${cells(stats)} |`); |
| 182 | } |
| 183 | console.log("\n## All cases\n"); |
| 184 | console.log( |
| 185 | `${header.replace("| case ", "")}\n| --- | ${Array.from({ length: 9 }, () => "---").join(" | ")} |`, |
| 186 | ); |
| 187 | const arms = [...totals].toSorted(([a], [b]) => b.localeCompare(a)); |
| 188 | for (const [arm, stats] of arms) console.log(`| ${arm} | ${cells(stats)} |`); |
| 189 | console.log(""); |
| 190 | for (const [arm, stats] of arms) { |
| 191 | if (Number.isNaN(stats.cost) || stats.pass === 0) continue; |
| 192 | console.log(`- ${arm}: cost per passing sample $${(stats.cost / stats.pass).toFixed(3)}`); |
| 193 | } |