The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
fix(evals): GPT evals use the configured Codex provider and CODEX_HOME (#38573)
Julius Marminge committed
Nov 19, 2026 at 12:15 UTC
56b077777468b247030bdb2ff9b0e42b58555520
4 files changed
+32
-12
evals/mcp-tool-use/README.md
+1
-1
@@ -13,7 +13,7 @@ pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus
13
```
14
15
- `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator.
16
-- Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs.
16
+- Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the Codex model provider: `model_provider` in `$CODEX_HOME/config.toml` (default `~/.codex/config.toml`) when it names a `[model_providers.<name>]` table, otherwise `[model_providers.vibeproxy]`. That table needs `base_url` (the proxy's OpenAI-compatible URL, such as `http://localhost:8317/v1`) and a key in `experimental_bearer_token` or the variable named by `env_key`. Keep the proxy under about 12 samples in flight across all runs.
17
- `--shell-model` picks the model that plays the shell: `claude-haiku-5-5` by default, or `gpt-6.1-terra` / `gpt-6-luna` through the same proxy, so Codex runs need no Anthropic key. Prefer Terra: Luna often contradicts the case's world (it calls named checkouts missing or files empty). Compare runs only when they used the same shell model.
18
- Flags: `--case <ids>`, `--cases <file.ndjson>`, `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (samples in flight, default 8), `--max-steps <n>`, `--effort <level>`, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed).
19
- `--baseline <results.jsonl> --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run.
evals/mcp-tool-use/run.ts
+1
-1
@@ -2802,7 +2802,7 @@ export const summarize = (
2802
`- Pass: ${percent(passRate(rows))} of ${finished} samples.`,
2803
...(errors.length > 0 ? [errorNote] : []),
2804
codex
2805
- ? "- Cost: the agent is not priced. `vibeproxy` in `~/.codex/config.toml` may use a subscription or bill an API key per token; agent tokens are below."
2805
+ ? "- Cost: the agent is not priced. the Codex model provider (see README) may use a subscription or bill an API key per token; agent tokens are below."
2806
: `- Estimated cost $${cost.toFixed(2)}, $${(cost / rows.length).toFixed(3)} per sample.`,
2807
`- Agent tokens: ${agent.tokens}${codex ? "." : ` ($${agent.cost.toFixed(2)}).`}`,
2808
`- Shell simulator (${options.shellModel}) tokens: ${shell.tokens} (API-price estimate $${shell.cost.toFixed(2)}).`,
evals/shared/proxy.ts
+29
-9
@@ -5,24 +5,41 @@ import * as Schema from "effect/Schema";
5
6
// Read credentials into memory only. Python's TOML parser avoids another runner dependency.
7
const decodeProxyConfig = Schema.decodeUnknownSync(
8
- Schema.Struct({ url: Schema.String, headers: Schema.Record(Schema.String, Schema.String) }),
8
+ Schema.Union([
9
+ Schema.Struct({ error: Schema.String }),
10
+ Schema.Struct({ url: Schema.String, headers: Schema.Record(Schema.String, Schema.String) }),
11
+ ]),
12
);
10
-/** Reads the `vibeproxy` CLIProxyAPI provider from `~/.codex/config.toml`. */
11
-export const proxyConfig = () =>
12
- decodeProxyConfig(
13
+/**
14
+ * Reads the CLIProxyAPI provider from `$CODEX_HOME/config.toml` (default `~/.codex`): the
15
+ * configured `model_provider` when it is a custom provider, otherwise `vibeproxy`.
16
+ */
17
+export const proxyConfig = () => {
18
+ const config = decodeProxyConfig(
19
JSON.parse(
20
NodeChildProcess.execFileSync(
21
"python3",
22
[
23
"-c",
24
`
19
-import json, os, pathlib, tomllib
20
-home = pathlib.Path.home() / '.codex'
21
-p = tomllib.loads((home / 'config.toml').read_text())['model_providers']['vibeproxy']
25
+import json, os, pathlib, sys, tomllib
26
+home = pathlib.Path(os.environ.get('CODEX_HOME') or pathlib.Path.home() / '.codex')
27
+def fail(message):
28
+ print(json.dumps({'error': message + ' The GPT evals call models through a CLIProxyAPI server set up as a custom model provider in Codex: set model_provider = "<name>" and add [model_providers.<name>] with base_url and its key.'}))
29
+ sys.exit()
30
+path = home / 'config.toml'
31
+if not path.is_file(): fail(f'No Codex config at {path}.')
32
+config = tomllib.loads(path.read_text())
33
+providers = config.get('model_providers', {})
34
+name = config.get('model_provider') if config.get('model_provider') in providers else 'vibeproxy'
35
+if name not in providers: fail(f'{path} has no custom model provider to use (model_provider is {config.get("model_provider", "unset")!r}).')
36
+p = providers[name]
37
+if not p.get('base_url'): fail(f'[model_providers.{name}] in {path} has no base_url.')
38
h = p.get('http_headers', {}).copy()
39
h.update({k: os.environ[v] for k,v in p.get('env_http_headers', {}).items()})
24
-key = os.environ[p['env_key']] if p.get('env_key') else p.get('experimental_bearer_token')
25
-if not key and p.get('requires_openai_auth'): key = json.loads((home / 'auth.json').read_text()).get('OPENAI_API_KEY')
40
+key = os.environ.get(p['env_key']) if p.get('env_key') else p.get('experimental_bearer_token')
41
+if p.get('env_key') and not key: fail(f'[model_providers.{name}] in {path} reads its key from {p["env_key"]}, which is not set.')
42
+if not key and p.get('requires_openai_auth') and (home / 'auth.json').is_file(): key = json.loads((home / 'auth.json').read_text()).get('OPENAI_API_KEY')
43
if key: h['Authorization'] = 'Bearer ' + key
44
print(json.dumps({'url': p['base_url'].rstrip('/'), 'headers': h}))
45
`,
@@ -31,3 +48,6 @@ print(json.dumps({'url': p['base_url'].rstrip('/'), 'headers': h}))
48
),
49
),
50
);
51
+ if ("error" in config) throw new Error(config.error);
52
+ return config;
53
+};
evals/test-signal/README.md
+1
-1
@@ -12,7 +12,7 @@ pnpm test-signal --out runs/baseline
12
pnpm test-signal --out runs/cut --prompt /path/to/variant.md --case new-table-row
13
```
14
15
-- Models go through the `vibeproxy` CLIProxyAPI provider in `~/.codex/config.toml`. Default models: `gpt-6.1-terra` and `gpt-6-luna`. Pass `--model` once per model to change them.
15
+- Models go through a CLIProxyAPI server, set up as the Codex model provider: `model_provider` in `$CODEX_HOME/config.toml` (default `~/.codex/config.toml`) when it names a `[model_providers.<name>]` table, otherwise `[model_providers.vibeproxy]`. That table needs `base_url` (the proxy's OpenAI-compatible URL, such as `http://localhost:8317/v1`) and a key in `experimental_bearer_token` or the variable named by `env_key`. Default models: `gpt-6.1-terra` and `gpt-6-luna`. Pass `--model` once per model to change them.
16
- Flags: `--prompt <file>` (a variant of the check, default the real one), `--case <id>` (repeatable), `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (default 6), `--reasoning <level>` (default `low`, as the check runs).
17
- A bad case passes when a comment lands inside a target test. A guard case passes when the check posts nothing and replies exactly `All clear`.
18
- `summary.md` gives the prompt hash and word count, then per model: bad cases caught, guards clean, errors, and tokens.