The best way to control Claude Code, Codex and more from any computer or phone

main
ts 193 lines 6.67 KB
Raw
1 #!/usr/bin/env node
2 // @effect-diagnostics nodeBuiltinImport:off globalConsole:off
3 /**
4 * `pnpm mcp-tool-use:compare <results.jsonl>... [--model <id>]`: per case and arm, pass rate and
5 * forge spend. Rows are regraded with this checkout's cases. Cache writes are priced as reads, so
6 * run order does not pick the cheaper arm; only Claude agents are priced.
7 */
8 import * as NodeFS from "node:fs";
9 import * as NodeUtil from "node:util";
10
11 import * as Schema from "effect/Schema";
12
13 import {
14 gradeSample,
15 McpToolUseCase,
16 modelRequests,
17 pastedText,
18 PRICES,
19 worldFilePaths,
20 } from "./run.ts";
21
22 const { values, positionals } = NodeUtil.parseArgs({
23 allowPositionals: true,
24 options: {
25 model: { type: "string", default: "claude-sonnet-5-5" },
26 // Rows from before shellModel was recorded: the shell model they ran with (the runner's default).
27 "shell-model": { type: "string", default: "claude-haiku-5-5" },
28 cases: {
29 type: "string",
30 default: new URL("./cases/pr-tools-vs-cli.ndjson", import.meta.url).pathname,
31 },
32 },
33 });
34 const decodeCase = Schema.decodeUnknownSync(Schema.fromJsonString(McpToolUseCase));
35 const cases = new Map(
36 NodeFS.readFileSync(values.cases, "utf8")
37 .split("\n")
38 .filter((line) => line.trim() !== "")
39 .map((line) => decodeCase(line))
40 .map((testCase) => [testCase.id, testCase]),
41 );
42
43 type Usage = { input: number; output: number; cacheWrite: number; cacheRead: number };
44 type Call = { step: number; tool: string; input: unknown; failed?: boolean; output?: string };
45 type Row = {
46 id: string;
47 arm?: string;
48 pass: boolean;
49 error?: string;
50 stopReason?: string;
51 calls?: ReadonlyArray<Call>;
52 modelCalls?: number;
53 shellModel?: string;
54 usage: Usage;
55 shellUsage: Usage;
56 };
57
58 const FORGE_TOOL =
59 /^(pull_request_\w+|ci_logs|ci_rerun|(un)?watch_pull_request|(un)?link_pull_requests?|list_thread_pull_requests)$/;
60 const FORGE_COMMAND = /(^|[\s;&|(`$])(gh|cafe|glab|tea|src)\s|api\.github\.com|git\.cafe\/api/;
61 const command = (call: Call) =>
62 call.tool === "Bash" && typeof call.input === "object" && call.input !== null
63 ? String((call.input as { command?: unknown }).command ?? "")
64 : "";
65 const isForge = (call: Call) => FORGE_TOOL.test(call.tool) || FORGE_COMMAND.test(command(call));
66 const sleepSeconds = (call: Call) =>
67 [...command(call).matchAll(/\bsleep\s+(\d+)/g)].reduce((sum, [, s]) => sum + Number(s), 0) +
68 (call.tool === "sleep" || call.tool === "Monitor" ? 60 : 0);
69
70 const price = (usage: Usage, model: string) => {
71 const rates = (PRICES as Record<string, Usage | undefined>)[model];
72 if (rates === undefined) return Number.NaN;
73 return (
74 (usage.input * rates.input +
75 usage.output * rates.output +
76 usage.cacheWrite * rates.cacheRead +
77 usage.cacheRead * rates.cacheRead) /
78 1e6
79 );
80 };
81
82 const graded = positionals.flatMap((file) =>
83 NodeFS.readFileSync(file, "utf8")
84 .split("\n")
85 .filter((line) => line.trim() !== "")
86 .map((line) => JSON.parse(line) as Row)
87 .filter((row) => row.error === undefined)
88 .map((row) => {
89 const testCase = cases.get(row.id);
90 if (testCase === undefined) return undefined;
91 const expect = row.arm === "raw" ? (testCase.expectRaw ?? testCase.expect) : testCase.expect;
92 const checks = gradeSample(
93 expect,
94 row as unknown as Parameters<typeof gradeSample>[1],
95 pastedText(testCase),
96 worldFilePaths(testCase),
97 );
98 return { ...row, pass: checks.every((check) => check.pass) };
99 }),
100 );
101 // A row whose case is not in --cases cannot be regraded, so it stays out of every number.
102 const rows: Array<Row> = graded.filter((row) => row !== undefined);
103 const unmatched = graded.length - rows.length;
104
105 type Stats = {
106 n: number;
107 pass: number;
108 steps: number;
109 forge: number;
110 failed: number;
111 bytes: number;
112 sleep: number;
113 help: number;
114 tokens: number;
115 cost: number;
116 };
117 const empty = (): Stats => ({
118 n: 0,
119 pass: 0,
120 steps: 0,
121 forge: 0,
122 failed: 0,
123 bytes: 0,
124 sleep: 0,
125 help: 0,
126 tokens: 0,
127 cost: 0,
128 });
129 const add = (stats: Stats, row: Row) => {
130 const calls = row.calls ?? [];
131 const forge = calls.filter(isForge);
132 stats.n++;
133 if (row.pass) stats.pass++;
134 stats.steps += modelRequests(row);
135 stats.forge += forge.length;
136 stats.failed += forge.filter((call) => call.failed === true).length;
137 stats.bytes += forge.reduce((sum, call) => sum + (call.output?.length ?? 0), 0);
138 stats.sleep += calls.reduce((sum, call) => sum + sleepSeconds(call), 0);
139 stats.help += forge.filter((call) => /--help\b|\shelp\s*$/.test(command(call))).length;
140 stats.tokens += row.usage.input + row.usage.cacheWrite + row.usage.cacheRead + row.usage.output;
141 stats.cost +=
142 price(row.usage, values.model) + price(row.shellUsage, row.shellModel ?? values["shell-model"]);
143 };
144
145 const byCase = new Map<string, Map<string, Stats>>();
146 const totals = new Map<string, Stats>();
147 for (const row of rows) {
148 const arm = row.arm ?? "tools";
149 const perArm = byCase.get(row.id) ?? new Map<string, Stats>();
150 byCase.set(row.id, perArm);
151 const stats = perArm.get(arm) ?? empty();
152 perArm.set(arm, stats);
153 add(stats, row);
154 const total = totals.get(arm) ?? empty();
155 totals.set(arm, total);
156 add(total, row);
157 }
158
159 const mean = (value: number, n: number, digits = 1) =>
160 n === 0 ? "-" : (value / n).toFixed(digits);
161 const cells = (s: Stats) =>
162 [
163 `${s.pass}/${s.n}`,
164 mean(s.steps, s.n),
165 mean(s.forge, s.n),
166 mean(s.failed, s.n),
167 mean(s.bytes / 1024, s.n),
168 mean(s.sleep, s.n, 0),
169 mean(s.help, s.n),
170 mean(s.tokens / 1000, s.n, 0),
171 Number.isNaN(s.cost) ? "-" : `$${mean(s.cost, s.n, 3)}`,
172 ].join(" | ");
173 const header =
174 "| case | arm | pass | model calls | forge calls | failed | forge KB | sleep s | --help | ktokens | $/sample |";
175 console.log(
176 `# Tools vs raw CLI (${values.model})\n\nMeans per sample.${unmatched > 0 ? ` Left out ${unmatched} rows whose case is not in ${values.cases}.` : ""}\n\n${header}`,
177 );
178 console.log(`| --- | --- | ${Array.from({ length: 9 }, () => "---").join(" | ")} |`);
179 for (const [id, perArm] of [...byCase].toSorted(([a], [b]) => a.localeCompare(b))) {
180 for (const [arm, stats] of [...perArm].toSorted(([a], [b]) => b.localeCompare(a)))
181 console.log(`| ${id} | ${arm} | ${cells(stats)} |`);
182 }
183 console.log("\n## All cases\n");
184 console.log(
185 `${header.replace("| case ", "")}\n| --- | ${Array.from({ length: 9 }, () => "---").join(" | ")} |`,
186 );
187 const arms = [...totals].toSorted(([a], [b]) => b.localeCompare(a));
188 for (const [arm, stats] of arms) console.log(`| ${arm} | ${cells(stats)} |`);
189 console.log("");
190 for (const [arm, stats] of arms) {
191 if (Number.isNaN(stats.cost) || stats.pass === 0) continue;
192 console.log(`- ${arm}: cost per passing sample $${(stats.cost / stats.pass).toFixed(3)}`);
193 }