The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 14:30 UTC
chore(evals): store eval cases as .ndjson so review tools skip them (#25782)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 26, 2026 at 22:52 UTC
a9b513b50cc967cc715228b38cb922d1aaaa3a8b
5 files changed
+5
-5
evals/mcp-tool-use/README.md
+1
-1
@@ -14,7 +14,7 @@ pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus
14
15
- `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator.
16
- Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs.
17
-- Flags: `--case <ids>`, `--cases <file.jsonl>`, `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (samples in flight, default 8), `--max-steps <n>`, `--effort <level>`, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed).
17
+- Flags: `--case <ids>`, `--cases <file.ndjson>`, `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (samples in flight, default 8), `--max-steps <n>`, `--effort <level>`, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed).
18
- `--baseline <results.jsonl> --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run.
19
- Cost: a full Sonnet run is about $9, and a full Opus run about $17. Codex models through the proxy cost only the shell simulator.
20
evals/mcp-tool-use/cases/orchestration.ndjson
renamed
evals/mcp-tool-use/cases/pull-requests.ndjson
renamed
evals/mcp-tool-use/cases/visual-preview.ndjson
renamed
evals/mcp-tool-use/run.ts
+4
-4
@@ -1678,7 +1678,7 @@ const main = Effect.gen(function* () {
1678
}
1679
const fs = yield* FileSystem.FileSystem;
1680
const path = yield* Path.Path;
1681
- const readJsonl = <A, E>(file: string, decode: (line: string) => Effect.Effect<A, E>) =>
1681
+ const readNdjson = <A, E>(file: string, decode: (line: string) => Effect.Effect<A, E>) =>
1682
fs.readFileString(file).pipe(
1683
Effect.map((content) => content.split("\n").filter((line) => line.trim() !== "")),
1684
Effect.flatMap((lines) => Effect.forEach(lines, (line) => decode(line))),
@@ -1687,10 +1687,10 @@ const main = Effect.gen(function* () {
1687
const caseFiles =
1688
flags.cases ??
1689
(yield* fs.readDirectory(casesDir))
1690
- .filter((name) => name.endsWith(".jsonl"))
1690
+ .filter((name) => name.endsWith(".ndjson"))
1691
.toSorted()
1692
.map((name) => path.join(casesDir, name));
1693
- const cases = (yield* Effect.forEach(caseFiles, (file) => readJsonl(file, decodeCase))).flat();
1693
+ const cases = (yield* Effect.forEach(caseFiles, (file) => readNdjson(file, decodeCase))).flat();
1694
const wanted = flags.case?.flatMap((ids) => ids.split(","));
1695
const codex = isCodex(flags.model);
1696
const harness = codex ? "codex" : "claude";
@@ -1764,7 +1764,7 @@ const main = Effect.gen(function* () {
1764
shellModel: flags["shell-model"],
1765
};
1766
const previous =
1767
- flags.baseline === undefined ? [] : yield* readJsonl(flags.baseline, decodeBaselineRow);
1767
+ flags.baseline === undefined ? [] : yield* readNdjson(flags.baseline, decodeBaselineRow);
1768
const baseline = new Map(groups(previous, (row) => row.id).map(([id, r]) => [id, passRate(r)]));
1769
// Only a case that passed every baseline sample can stop a run; flaky ones are just reported.
1770
const passedBefore = (id: string) => baseline.get(id) === 1;