@devries-cameron20 / t3code Mirror · read-only / commits / 56b0777774
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

fix(evals): GPT evals use the configured Codex provider and CODEX_HOME (#38573)

Julius Marminge committed Nov 19, 2026 at 12:15 UTC 56b077777468b247030bdb2ff9b0e42b58555520
4 files changed +32 -12
evals/mcp-tool-use/README.md
+1 -1
@@ -13,7 +13,7 @@ pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus
13 ```
14
15 - `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator.
16 -- Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs.
16 +- Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the Codex model provider: `model_provider` in `$CODEX_HOME/config.toml` (default `~/.codex/config.toml`) when it names a `[model_providers.<name>]` table, otherwise `[model_providers.vibeproxy]`. That table needs `base_url` (the proxy's OpenAI-compatible URL, such as `http://localhost:8317/v1`) and a key in `experimental_bearer_token` or the variable named by `env_key`. Keep the proxy under about 12 samples in flight across all runs.
17 - `--shell-model` picks the model that plays the shell: `claude-haiku-5-5` by default, or `gpt-6.1-terra` / `gpt-6-luna` through the same proxy, so Codex runs need no Anthropic key. Prefer Terra: Luna often contradicts the case's world (it calls named checkouts missing or files empty). Compare runs only when they used the same shell model.
18 - Flags: `--case <ids>`, `--cases <file.ndjson>`, `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (samples in flight, default 8), `--max-steps <n>`, `--effort <level>`, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed).
19 - `--baseline <results.jsonl> --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run.
evals/mcp-tool-use/run.ts
+1 -1
@@ -2802,7 +2802,7 @@ export const summarize = (
2802 `- Pass: ${percent(passRate(rows))} of ${finished} samples.`,
2803 ...(errors.length > 0 ? [errorNote] : []),
2804 codex
2805 - ? "- Cost: the agent is not priced. `vibeproxy` in `~/.codex/config.toml` may use a subscription or bill an API key per token; agent tokens are below."
2805 + ? "- Cost: the agent is not priced. the Codex model provider (see README) may use a subscription or bill an API key per token; agent tokens are below."
2806 : `- Estimated cost $${cost.toFixed(2)}, $${(cost / rows.length).toFixed(3)} per sample.`,
2807 `- Agent tokens: ${agent.tokens}${codex ? "." : ` ($${agent.cost.toFixed(2)}).`}`,
2808 `- Shell simulator (${options.shellModel}) tokens: ${shell.tokens} (API-price estimate $${shell.cost.toFixed(2)}).`,
evals/shared/proxy.ts
+29 -9
@@ -5,24 +5,41 @@ import * as Schema from "effect/Schema";
5
6 // Read credentials into memory only. Python's TOML parser avoids another runner dependency.
7 const decodeProxyConfig = Schema.decodeUnknownSync(
8 - Schema.Struct({ url: Schema.String, headers: Schema.Record(Schema.String, Schema.String) }),
8 + Schema.Union([
9 + Schema.Struct({ error: Schema.String }),
10 + Schema.Struct({ url: Schema.String, headers: Schema.Record(Schema.String, Schema.String) }),
11 + ]),
12 );
10 -/** Reads the `vibeproxy` CLIProxyAPI provider from `~/.codex/config.toml`. */
11 -export const proxyConfig = () =>
12 - decodeProxyConfig(
13 +/**
14 + * Reads the CLIProxyAPI provider from `$CODEX_HOME/config.toml` (default `~/.codex`): the
15 + * configured `model_provider` when it is a custom provider, otherwise `vibeproxy`.
16 + */
17 +export const proxyConfig = () => {
18 + const config = decodeProxyConfig(
19 JSON.parse(
20 NodeChildProcess.execFileSync(
21 "python3",
22 [
23 "-c",
24 `
19 -import json, os, pathlib, tomllib
20 -home = pathlib.Path.home() / '.codex'
21 -p = tomllib.loads((home / 'config.toml').read_text())['model_providers']['vibeproxy']
25 +import json, os, pathlib, sys, tomllib
26 +home = pathlib.Path(os.environ.get('CODEX_HOME') or pathlib.Path.home() / '.codex')
27 +def fail(message):
28 + print(json.dumps({'error': message + ' The GPT evals call models through a CLIProxyAPI server set up as a custom model provider in Codex: set model_provider = "<name>" and add [model_providers.<name>] with base_url and its key.'}))
29 + sys.exit()
30 +path = home / 'config.toml'
31 +if not path.is_file(): fail(f'No Codex config at {path}.')
32 +config = tomllib.loads(path.read_text())
33 +providers = config.get('model_providers', {})
34 +name = config.get('model_provider') if config.get('model_provider') in providers else 'vibeproxy'
35 +if name not in providers: fail(f'{path} has no custom model provider to use (model_provider is {config.get("model_provider", "unset")!r}).')
36 +p = providers[name]
37 +if not p.get('base_url'): fail(f'[model_providers.{name}] in {path} has no base_url.')
38 h = p.get('http_headers', {}).copy()
39 h.update({k: os.environ[v] for k,v in p.get('env_http_headers', {}).items()})
24 -key = os.environ[p['env_key']] if p.get('env_key') else p.get('experimental_bearer_token')
25 -if not key and p.get('requires_openai_auth'): key = json.loads((home / 'auth.json').read_text()).get('OPENAI_API_KEY')
40 +key = os.environ.get(p['env_key']) if p.get('env_key') else p.get('experimental_bearer_token')
41 +if p.get('env_key') and not key: fail(f'[model_providers.{name}] in {path} reads its key from {p["env_key"]}, which is not set.')
42 +if not key and p.get('requires_openai_auth') and (home / 'auth.json').is_file(): key = json.loads((home / 'auth.json').read_text()).get('OPENAI_API_KEY')
43 if key: h['Authorization'] = 'Bearer ' + key
44 print(json.dumps({'url': p['base_url'].rstrip('/'), 'headers': h}))
45 `,
@@ -31,3 +48,6 @@ print(json.dumps({'url': p['base_url'].rstrip('/'), 'headers': h}))
48 ),
49 ),
50 );
51 + if ("error" in config) throw new Error(config.error);
52 + return config;
53 +};
evals/test-signal/README.md
+1 -1
@@ -12,7 +12,7 @@ pnpm test-signal --out runs/baseline
12 pnpm test-signal --out runs/cut --prompt /path/to/variant.md --case new-table-row
13 ```
14
15 -- Models go through the `vibeproxy` CLIProxyAPI provider in `~/.codex/config.toml`. Default models: `gpt-6.1-terra` and `gpt-6-luna`. Pass `--model` once per model to change them.
15 +- Models go through a CLIProxyAPI server, set up as the Codex model provider: `model_provider` in `$CODEX_HOME/config.toml` (default `~/.codex/config.toml`) when it names a `[model_providers.<name>]` table, otherwise `[model_providers.vibeproxy]`. That table needs `base_url` (the proxy's OpenAI-compatible URL, such as `http://localhost:8317/v1`) and a key in `experimental_bearer_token` or the variable named by `env_key`. Default models: `gpt-6.1-terra` and `gpt-6-luna`. Pass `--model` once per model to change them.
16 - Flags: `--prompt <file>` (a variant of the check, default the real one), `--case <id>` (repeatable), `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (default 6), `--reasoning <level>` (default `low`, as the check runs).
17 - A bad case passes when a comment lands inside a target test. A guard case passes when the check posts nothing and replies exactly `All clear`.
18 - `summary.md` gives the prompt hash and word count, then per model: bad cases caught, guards clean, errors, and tokens.