@devries-cameron20 / t3code Mirror · read-only / commits / a9b513b50c
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 14:30 UTC

chore(evals): store eval cases as .ndjson so review tools skip them (#25782)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 26, 2026 at 22:52 UTC a9b513b50cc967cc715228b38cb922d1aaaa3a8b
5 files changed +5 -5
evals/mcp-tool-use/README.md
+1 -1
@@ -14,7 +14,7 @@ pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus
14
15 - `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator.
16 - Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs.
17 -- Flags: `--case <ids>`, `--cases <file.jsonl>`, `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (samples in flight, default 8), `--max-steps <n>`, `--effort <level>`, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed).
17 +- Flags: `--case <ids>`, `--cases <file.ndjson>`, `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (samples in flight, default 8), `--max-steps <n>`, `--effort <level>`, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed).
18 - `--baseline <results.jsonl> --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run.
19 - Cost: a full Sonnet run is about $9, and a full Opus run about $17. Codex models through the proxy cost only the shell simulator.
20
evals/mcp-tool-use/cases/orchestration.ndjson renamed
evals/mcp-tool-use/cases/pull-requests.ndjson renamed
evals/mcp-tool-use/cases/visual-preview.ndjson renamed
evals/mcp-tool-use/run.ts
+4 -4
@@ -1678,7 +1678,7 @@ const main = Effect.gen(function* () {
1678 }
1679 const fs = yield* FileSystem.FileSystem;
1680 const path = yield* Path.Path;
1681 - const readJsonl = <A, E>(file: string, decode: (line: string) => Effect.Effect<A, E>) =>
1681 + const readNdjson = <A, E>(file: string, decode: (line: string) => Effect.Effect<A, E>) =>
1682 fs.readFileString(file).pipe(
1683 Effect.map((content) => content.split("\n").filter((line) => line.trim() !== "")),
1684 Effect.flatMap((lines) => Effect.forEach(lines, (line) => decode(line))),
@@ -1687,10 +1687,10 @@ const main = Effect.gen(function* () {
1687 const caseFiles =
1688 flags.cases ??
1689 (yield* fs.readDirectory(casesDir))
1690 - .filter((name) => name.endsWith(".jsonl"))
1690 + .filter((name) => name.endsWith(".ndjson"))
1691 .toSorted()
1692 .map((name) => path.join(casesDir, name));
1693 - const cases = (yield* Effect.forEach(caseFiles, (file) => readJsonl(file, decodeCase))).flat();
1693 + const cases = (yield* Effect.forEach(caseFiles, (file) => readNdjson(file, decodeCase))).flat();
1694 const wanted = flags.case?.flatMap((ids) => ids.split(","));
1695 const codex = isCodex(flags.model);
1696 const harness = codex ? "codex" : "claude";
@@ -1764,7 +1764,7 @@ const main = Effect.gen(function* () {
1764 shellModel: flags["shell-model"],
1765 };
1766 const previous =
1767 - flags.baseline === undefined ? [] : yield* readJsonl(flags.baseline, decodeBaselineRow);
1767 + flags.baseline === undefined ? [] : yield* readNdjson(flags.baseline, decodeBaselineRow);
1768 const baseline = new Map(groups(previous, (row) => row.id).map(([id, r]) => [id, passRate(r)]));
1769 // Only a case that passed every baseline sample can stop a run; flaky ones are just reported.
1770 const passedBefore = (id: string) => baseline.get(id) === 1;