@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 61 lines 2.65 KB
Raw
1 #!/usr/bin/env node
2 // @effect-diagnostics nodeBuiltinImport:off globalConsole:off
3 /**
4 * Grades saved samples again with this checkout's cases and graders: `pnpm mcp-tool-use:regrade
5 * <results.jsonl>... [--cases <file>]` prints pass counts per case, old → new. Rows with an error
6 * are skipped. A sample that stopped early and passes now is "cut short": its turn never finished.
7 */
8 import * as NodeFS from "node:fs";
9 import * as NodeURL from "node:url";
10 import * as NodeUtil from "node:util";
11 import * as Schema from "effect/Schema";
12
13 import { gradeSample, McpToolUseCase, pastedText, worldFilePaths } from "./run.ts";
14
15 const { values, positionals } = NodeUtil.parseArgs({
16 allowPositionals: true,
17 options: { cases: { type: "string", multiple: true } },
18 });
19 const caseFiles =
20 values.cases ??
21 NodeFS.readdirSync(new URL("./cases", import.meta.url))
22 .filter((name) => name.endsWith(".ndjson"))
23 .map((name) => NodeURL.fileURLToPath(new URL(`./cases/${name}`, import.meta.url)));
24 const decodeCase = Schema.decodeUnknownSync(Schema.fromJsonString(McpToolUseCase));
25 const lines = (file: string) =>
26 NodeFS.readFileSync(file, "utf8")
27 .split("\n")
28 .filter((line) => line.trim() !== "");
29 const cases = new Map(
30 caseFiles.flatMap(lines).map((line) => [decodeCase(line).id, decodeCase(line)]),
31 );
32
33 for (const file of positionals) {
34 const counts = new Map<string, { n: number; was: number; now: number; cut: number }>();
35 for (const line of lines(file)) {
36 const row = JSON.parse(line);
37 const testCase = cases.get(row.id);
38 if (testCase === undefined || row.error !== undefined) continue;
39 const expect = row.arm === "raw" ? (testCase.expectRaw ?? testCase.expect) : testCase.expect;
40 const failed = gradeSample(expect, row, pastedText(testCase), worldFilePaths(testCase)).filter(
41 (check) => !check.pass,
42 );
43 const cut = failed.length === 0 && row.stopReason === "failed_early";
44 const pass = failed.length === 0 && !cut;
45 const count = counts.get(row.id) ?? { n: 0, was: 0, now: 0, cut: 0 };
46 count.n++;
47 if (row.pass) count.was++;
48 if (pass) count.now++;
49 if (cut) count.cut++;
50 counts.set(row.id, count);
51 if (pass !== row.pass || cut)
52 console.log(
53 `${row.id}#${count.n} ${row.pass ? "pass" : "FAIL"} → ${cut ? "cut short" : pass ? "pass" : "FAIL"}${failed.length > 0 ? `: ${failed.map((check) => check.check.slice(0, 100)).join(" | ")}` : ""}`,
54 );
55 }
56 console.log(`\n${file}\n| case | was | now |\n| --- | --- | --- |`);
57 for (const [id, { n, was, now, cut }] of counts)
58 console.log(
59 `| ${id} | ${was}/${n} | ${now}/${n}${cut > 0 ? ` (${cut} cut short)` : ""}${was === now ? "" : " *"} |`,
60 );
61 }