The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | #!/usr/bin/env node |
| 2 | // @effect-diagnostics nodeBuiltinImport:off globalConsole:off |
| 3 | /** |
| 4 | * Grades saved samples again with this checkout's cases and graders: `pnpm mcp-tool-use:regrade |
| 5 | * <results.jsonl>... [--cases <file>]` prints pass counts per case, old → new. Rows with an error |
| 6 | * are skipped. A sample that stopped early and passes now is "cut short": its turn never finished. |
| 7 | */ |
| 8 | import * as NodeFS from "node:fs"; |
| 9 | import * as NodeURL from "node:url"; |
| 10 | import * as NodeUtil from "node:util"; |
| 11 | import * as Schema from "effect/Schema"; |
| 12 | |
| 13 | import { gradeSample, McpToolUseCase, pastedText, worldFilePaths } from "./run.ts"; |
| 14 | |
| 15 | const { values, positionals } = NodeUtil.parseArgs({ |
| 16 | allowPositionals: true, |
| 17 | options: { cases: { type: "string", multiple: true } }, |
| 18 | }); |
| 19 | const caseFiles = |
| 20 | values.cases ?? |
| 21 | NodeFS.readdirSync(new URL("./cases", import.meta.url)) |
| 22 | .filter((name) => name.endsWith(".ndjson")) |
| 23 | .map((name) => NodeURL.fileURLToPath(new URL(`./cases/${name}`, import.meta.url))); |
| 24 | const decodeCase = Schema.decodeUnknownSync(Schema.fromJsonString(McpToolUseCase)); |
| 25 | const lines = (file: string) => |
| 26 | NodeFS.readFileSync(file, "utf8") |
| 27 | .split("\n") |
| 28 | .filter((line) => line.trim() !== ""); |
| 29 | const cases = new Map( |
| 30 | caseFiles.flatMap(lines).map((line) => [decodeCase(line).id, decodeCase(line)]), |
| 31 | ); |
| 32 | |
| 33 | for (const file of positionals) { |
| 34 | const counts = new Map<string, { n: number; was: number; now: number; cut: number }>(); |
| 35 | for (const line of lines(file)) { |
| 36 | const row = JSON.parse(line); |
| 37 | const testCase = cases.get(row.id); |
| 38 | if (testCase === undefined || row.error !== undefined) continue; |
| 39 | const expect = row.arm === "raw" ? (testCase.expectRaw ?? testCase.expect) : testCase.expect; |
| 40 | const failed = gradeSample(expect, row, pastedText(testCase), worldFilePaths(testCase)).filter( |
| 41 | (check) => !check.pass, |
| 42 | ); |
| 43 | const cut = failed.length === 0 && row.stopReason === "failed_early"; |
| 44 | const pass = failed.length === 0 && !cut; |
| 45 | const count = counts.get(row.id) ?? { n: 0, was: 0, now: 0, cut: 0 }; |
| 46 | count.n++; |
| 47 | if (row.pass) count.was++; |
| 48 | if (pass) count.now++; |
| 49 | if (cut) count.cut++; |
| 50 | counts.set(row.id, count); |
| 51 | if (pass !== row.pass || cut) |
| 52 | console.log( |
| 53 | `${row.id}#${count.n} ${row.pass ? "pass" : "FAIL"} → ${cut ? "cut short" : pass ? "pass" : "FAIL"}${failed.length > 0 ? `: ${failed.map((check) => check.check.slice(0, 100)).join(" | ")}` : ""}`, |
| 54 | ); |
| 55 | } |
| 56 | console.log(`\n${file}\n| case | was | now |\n| --- | --- | --- |`); |
| 57 | for (const [id, { n, was, now, cut }] of counts) |
| 58 | console.log( |
| 59 | `| ${id} | ${was}/${n} | ${now}/${n}${cut > 0 ? ` (${cut} cut short)` : ""}${was === now ? "" : " *"} |`, |
| 60 | ); |
| 61 | } |