The best way to control Claude Code, Codex and more from any computer or phone
fix(evals): GPT evals stop at startup when OpenAI auth is required but no key is found (#39384)
Julius Marminge committed
Nov 20, 2026 at 15:00 UTC
6788be775bb9ada8f6be49768269638619d600ac
2 files changed
+33
evals/shared/proxy.test.ts
new
+32
@@ -0,0 +1,32 @@
1
+// @effect-diagnostics nodeBuiltinImport:off
2
+import * as NodeFS from "node:fs";
3
+import * as NodeOS from "node:os";
4
+import * as NodePath from "node:path";
5
+import { describe, expect, it, vi } from "vite-plus/test";
6
+import { proxyConfig } from "./proxy.ts";
7
+
8
+const config = `model_provider = "cliproxy"
9
+
10
+[model_providers.cliproxy]
11
+base_url = "https://proxy.example/v1"
12
+requires_openai_auth = true
13
+`;
14
+
15
+describe("proxyConfig", () => {
16
+ it.each([
17
+ ["no auth.json", undefined],
18
+ ["a ChatGPT-login auth.json", { OPENAI_API_KEY: null, tokens: { access_token: "t" } }],
19
+ ])("fails when the provider requires OpenAI auth and %s has no key", (_, auth) => {
20
+ const home = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-evals-proxy-"));
21
+ using _home = {
22
+ [Symbol.dispose]: () => {
23
+ vi.unstubAllEnvs();
24
+ NodeFS.rmSync(home, { recursive: true });
25
+ },
26
+ };
27
+ NodeFS.writeFileSync(NodePath.join(home, "config.toml"), config);
28
+ if (auth) NodeFS.writeFileSync(NodePath.join(home, "auth.json"), JSON.stringify(auth));
29
+ vi.stubEnv("CODEX_HOME", home);
30
+ expect(() => proxyConfig()).toThrow(/requires_openai_auth/);
31
+ });
32
+});
evals/shared/proxy.ts
+1
@@ -40,6 +40,7 @@ h.update({k: os.environ[v] for k,v in p.get('env_http_headers', {}).items()})
40
key = os.environ.get(p['env_key']) if p.get('env_key') else p.get('experimental_bearer_token')
41
if p.get('env_key') and not key: fail(f'[model_providers.{name}] in {path} reads its key from {p["env_key"]}, which is not set.')
42
if not key and p.get('requires_openai_auth') and (home / 'auth.json').is_file(): key = json.loads((home / 'auth.json').read_text()).get('OPENAI_API_KEY')
43
+if not key and p.get('requires_openai_auth'): fail(f'[model_providers.{name}] in {path} sets requires_openai_auth, but {home / "auth.json"} has no OPENAI_API_KEY.')
44
if key: h['Authorization'] = 'Bearer ' + key
45
print(json.dumps({'url': p['base_url'].rstrip('/'), 'headers': h}))
46
`,