The best way to control Claude Code, Codex and more from any computer or phone

main
ts 541 lines 19.2 KB
Raw
1 import { assert, describe, it } from "@effect/vitest";
2 import { type ModelSelection, ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts";
3
4 import { resolveProviderCatalog, type ModelManifestData } from "./ModelManifest.ts";
5 import {
6 BUNDLED_CLAUDE_MODEL_CATALOG,
7 capClaudeCatalogEffort,
8 type ClaudeModelCatalog,
9 formatClaudeVersionUpgradeMessage,
10 getClaudeCatalogModelCapabilities,
11 normalizeClaudeCatalogEffort,
12 resolveClaudeCatalogApiModelId,
13 resolveClaudeCatalogContextWindowTokens,
14 resolveClaudeCatalogEffort,
15 resolveClaudeModelCatalog,
16 resolveClaudeModelsForVersion,
17 resolveClaudeModelSlug,
18 resolveClaudeUpdateRequiredModels,
19 scopeClaudeModelCatalog,
20 withClaudeCodeModelRoutes,
21 } from "./ClaudeModelCatalog.ts";
22
23 const manifest = (): ModelManifestData => ({
24 version: 1,
25 currentModels: {},
26 providers: {
27 claudeAgent: {
28 profiles: {
29 synthetic: {
30 capabilities: {
31 optionDescriptors: [
32 {
33 id: "effort",
34 label: "Reasoning",
35 type: "select",
36 options: [{ id: "extreme", label: "Extreme", isDefault: true }],
37 },
38 {
39 id: "contextWindow",
40 label: "Context Window",
41 type: "select",
42 options: [
43 { id: "large", label: "Large", isDefault: true },
44 { id: "small", label: "Small" },
45 ],
46 },
47 ],
48 },
49 adapter: {
50 claudeCode: {
51 effortMap: { extreme: "high" },
52 modelSuffixes: { contextWindow: { large: "[large]" } },
53 },
54 },
55 },
56 },
57 models: [
58 {
59 slug: "claude-synthetic-next",
60 name: "Claude Synthetic Next",
61 aliases: ["synthetic"],
62 status: "current",
63 profile: "synthetic",
64 adapter: { claudeCode: { minVersion: "3.2.0" } },
65 },
66 ],
67 },
68 },
69 });
70
71 const CLAUDE = ProviderDriverKind.make("claudeAgent");
72
73 const catalogFromManifest = (data: ModelManifestData) =>
74 resolveClaudeModelCatalog(resolveProviderCatalog(data, CLAUDE) ?? undefined);
75
76 describe("Claude model catalog", () => {
77 it("resolves capacity from selected options and fixed catalog windows without guessing custom models", () => {
78 const source = manifest();
79 const profile = source.providers!.claudeAgent!.profiles.synthetic!;
80 const catalog = catalogFromManifest({
81 ...source,
82 providers: {
83 claudeAgent: {
84 ...source.providers!.claudeAgent!,
85 profiles: {
86 fixed: {
87 capabilities: { optionDescriptors: [] },
88 adapter: { claudeCode: { fixedContextWindowTokens: 64_000 } },
89 },
90 synthetic: {
91 ...profile,
92 adapter: { claudeCode: { contextWindowTokens: { large: 1_000_000, small: 32_000 } } },
93 },
94 },
95 models: [
96 ...source.providers!.claudeAgent!.models,
97 {
98 slug: "fixed",
99 name: "Fixed",
100 status: "current",
101 profile: "fixed",
102 },
103 ],
104 },
105 },
106 });
107 const selection = { instanceId: ProviderInstanceId.make("claudeAgent"), model: "synthetic" };
108 assert.equal(resolveClaudeCatalogContextWindowTokens(catalog, selection), 1_000_000);
109 assert.equal(
110 resolveClaudeCatalogContextWindowTokens(catalog, {
111 ...selection,
112 options: [{ id: "contextWindow", value: "small" }],
113 }),
114 32_000,
115 );
116 assert.equal(
117 resolveClaudeCatalogContextWindowTokens(catalog, { ...selection, model: "fixed" }),
118 64_000,
119 );
120 assert.isUndefined(
121 resolveClaudeCatalogContextWindowTokens(catalog, { ...selection, model: "custom" }),
122 );
123 });
124
125 it("filters models at runtime-version boundaries and derives the upgrade message", () => {
126 const catalog = catalogFromManifest(manifest());
127 assert.deepStrictEqual(resolveClaudeModelsForVersion(catalog, "3.1.9"), []);
128 assert.deepStrictEqual(
129 resolveClaudeModelsForVersion(catalog, "3.2.0").map((model) => model.slug),
130 ["claude-synthetic-next"],
131 );
132 assert.deepStrictEqual(resolveClaudeUpdateRequiredModels(catalog, "3.1.9"), [
133 { slug: "claude-synthetic-next", name: "Claude Synthetic Next", minVersion: "3.2.0" },
134 ]);
135 assert.deepStrictEqual(resolveClaudeUpdateRequiredModels(catalog, "3.2.0"), []);
136 assert.strictEqual(
137 formatClaudeVersionUpgradeMessage(catalog, "3.1.9"),
138 "Claude Code v3.1.9 is too old for Claude Synthetic Next. Upgrade to v3.2.0 or newer to access it.",
139 );
140 // Claude Code before 2.1.321 does not know claude-fable-5-2 and runs it with a 200K window.
141 assert.notInclude(
142 resolveClaudeModelsForVersion(BUNDLED_CLAUDE_MODEL_CATALOG, "2.1.320").map(
143 (model) => model.slug,
144 ),
145 "claude-fable-5-2",
146 );
147 });
148
149 it("resolves aliases and declarative adapter mappings", () => {
150 const base = manifest();
151 const input: ModelManifestData = {
152 ...base,
153 providers: {
154 ...base.providers,
155 claudeAgent: {
156 ...base.providers!.claudeAgent!,
157 models: [
158 {
159 slug: "claude-synthetic-collision",
160 name: "Claude Synthetic Collision",
161 aliases: ["claude-synthetic-next"],
162 status: "current",
163 },
164 ...base.providers!.claudeAgent!.models,
165 ],
166 },
167 },
168 };
169 const catalog = catalogFromManifest(input);
170 assert.strictEqual(resolveClaudeModelSlug(catalog, "synthetic"), "claude-synthetic-next");
171 assert.strictEqual(
172 resolveClaudeModelSlug(catalog, "claude-synthetic-next"),
173 "claude-synthetic-next",
174 );
175 assert.strictEqual(normalizeClaudeCatalogEffort(catalog, "extreme", "synthetic"), "high");
176 assert.strictEqual(
177 resolveClaudeCatalogApiModelId(catalog, {
178 instanceId: ProviderInstanceId.make("claudeAgent"),
179 model: "synthetic",
180 }),
181 "claude-synthetic-next[large]",
182 );
183 });
184
185 it("appends custom models with their own descriptors and keeps bare slugs opaque", () => {
186 const catalog = scopeClaudeModelCatalog(catalogFromManifest(manifest()), [
187 "synthetic",
188 {
189 slug: "claude-custom-tuned",
190 name: "Tuned",
191 capabilities: {
192 optionDescriptors: [
193 {
194 id: "effort",
195 label: "Reasoning",
196 type: "select",
197 options: [
198 { id: "gentle", label: "Gentle", isDefault: true },
199 { id: "brutal", label: "Brutal" },
200 { id: "ultracode", label: "Ultracode" },
201 ],
202 },
203 ],
204 },
205 },
206 ]);
207
208 assert.strictEqual(resolveClaudeModelSlug(catalog, "synthetic"), "synthetic");
209 assert.isUndefined(resolveClaudeCatalogEffort(catalog, "synthetic", "extreme"));
210 assert.strictEqual(resolveClaudeCatalogEffort(catalog, "synthetic", "max"), "max");
211 assert.strictEqual(normalizeClaudeCatalogEffort(catalog, "max", "synthetic"), "max");
212
213 assert.strictEqual(
214 resolveClaudeCatalogEffort(catalog, "claude-custom-tuned", "brutal"),
215 "brutal",
216 );
217 assert.strictEqual(
218 resolveClaudeCatalogEffort(catalog, "claude-custom-tuned", "bogus"),
219 "gentle",
220 );
221 assert.deepStrictEqual(
222 getClaudeCatalogModelCapabilities(catalog, "claude-custom-tuned").optionDescriptors?.map(
223 ({ id, type }) => `${id}:${type}`,
224 ),
225 ["effort:select", "ultracode:boolean"],
226 );
227 assert.strictEqual(
228 normalizeClaudeCatalogEffort(catalog, "brutal", "claude-custom-tuned"),
229 "brutal",
230 );
231 assert.strictEqual(
232 resolveClaudeCatalogApiModelId(catalog, {
233 instanceId: ProviderInstanceId.make("claudeAgent"),
234 model: "claude-custom-tuned",
235 options: [{ id: "effort", value: "brutal" }],
236 }),
237 "claude-custom-tuned",
238 );
239 assert.deepStrictEqual(
240 resolveClaudeModelsForVersion(catalog, "3.2.0").map((model) => model.slug),
241 ["claude-synthetic-next", "claude-custom-tuned"],
242 );
243 });
244
245 // Rows Claude Code 2.1.310 reports from initialization for each configuration.
246 it("sends the model id Claude Code reports when it serves a catalog model under another id", () => {
247 const apiModel = (
248 catalog: ClaudeModelCatalog,
249 model: string,
250 options?: ModelSelection["options"],
251 ) =>
252 resolveClaudeCatalogApiModelId(catalog, {
253 instanceId: ProviderInstanceId.make("claudeAgent"),
254 model,
255 ...(options ? { options } : {}),
256 });
257
258 // Bedrock with ANTHROPIC_DEFAULT_*_MODEL and an availableModels allowlist of their ids.
259 const configuredBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [
260 { value: "default", resolvedModel: "global.anthropic.claude-opus-5-5" },
261 { value: "sonnet", resolvedModel: "global.anthropic.claude-sonnet-5-5[1m]" },
262 { value: "opus", resolvedModel: "global.anthropic.claude-opus-5-5" },
263 { value: "haiku", resolvedModel: "global.anthropic.claude-haiku-5-5" },
264 ]);
265 assert.strictEqual(apiModel(configuredBedrock, "claude-haiku-5-5"), "haiku");
266 assert.strictEqual(apiModel(configuredBedrock, "haiku-5.5"), "haiku");
267 assert.strictEqual(apiModel(configuredBedrock, "claude-sonnet-5-5"), "sonnet");
268 assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5-5"), "opus");
269 assert.strictEqual(apiModel(configuredBedrock, "claude-opus-5"), "claude-opus-5");
270
271 // Bedrock with no model settings: Claude Code's `haiku` is Haiku 4.5 there.
272 const defaultBedrock = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [
273 {
274 value: "us.anthropic.claude-opus-4-6-v1",
275 resolvedModel: "us.anthropic.claude-opus-4-6-v1",
276 },
277 {
278 value: "us.anthropic.claude-opus-4-6-v1[1m]",
279 resolvedModel: "us.anthropic.claude-opus-4-6-v1[1m]",
280 },
281 { value: "haiku", resolvedModel: "us.anthropic.claude-haiku-4-5-20251001-v1:0" },
282 ]);
283 assert.strictEqual(
284 apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "1m" }]),
285 "us.anthropic.claude-opus-4-6-v1[1m]",
286 );
287 assert.strictEqual(
288 apiModel(defaultBedrock, "claude-opus-4-6", [{ id: "contextWindow", value: "200k" }]),
289 "us.anthropic.claude-opus-4-6-v1",
290 );
291 assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-4-5"), "haiku");
292 assert.strictEqual(apiModel(defaultBedrock, "claude-haiku-5-5"), "claude-haiku-5-5");
293
294 // First-party ids already are what Claude Code runs, even when an alias points elsewhere.
295 const firstParty = withClaudeCodeModelRoutes(BUNDLED_CLAUDE_MODEL_CATALOG, [
296 { value: "default", resolvedModel: "claude-opus-5-5" },
297 { value: "opus", resolvedModel: "claude-opus-5-5" },
298 { value: "claude-fable-5-1[1m]", resolvedModel: "claude-fable-5-1[1m]" },
299 { value: "haiku", resolvedModel: "claude-haiku-4-5-20251001" },
300 ]);
301 assert.strictEqual(firstParty, BUNDLED_CLAUDE_MODEL_CATALOG);
302 });
303
304 it("marks effort options that would run above the cap without changing ids or defaults", () => {
305 const effortModel = (slug: string) => ({
306 slug,
307 name: slug,
308 isCustom: false,
309 capabilities: {
310 optionDescriptors: [
311 {
312 id: "effort",
313 label: "Reasoning",
314 type: "select" as const,
315 options: [
316 { id: "medium", label: "Medium" },
317 { id: "high", label: "High", isDefault: true },
318 { id: "xhigh", label: "Extra High" },
319 { id: "max", label: "Max" },
320 { id: "ultracode", label: "Ultracode" },
321 { id: "ultrathink", label: "Ultrathink" },
322 ],
323 },
324 ],
325 },
326 });
327 const models = [effortModel("claude-plain"), effortModel("claude-remapped")];
328 const catalog: ClaudeModelCatalog = {
329 models: [
330 {
331 model: models[0]!,
332 runtime: { effortMap: { ultracode: "xhigh", ultrathink: null } },
333 compatibility: {},
334 },
335 {
336 model: models[1]!,
337 runtime: { effortMap: { max: "high", xhigh: "max", ultrathink: null } },
338 compatibility: {},
339 },
340 ],
341 };
342 const options = (cap: "high" | "xhigh") =>
343 capClaudeCatalogEffort(catalog, models, [{ maxEffortLevel: cap }]).map((model) => {
344 const effort = model.capabilities?.optionDescriptors?.[0];
345 return effort?.type === "select" ? effort.options : undefined;
346 });
347
348 assert.deepStrictEqual(options("high"), [
349 [
350 { id: "medium", label: "Medium" },
351 { id: "high", label: "High", isDefault: true },
352 { id: "xhigh", label: "Extra High (capped to High)" },
353 { id: "max", label: "Max (capped to High)" },
354 { id: "ultracode", label: "Ultracode (capped to High)" },
355 { id: "ultrathink", label: "Ultrathink" },
356 ],
357 [
358 { id: "medium", label: "Medium" },
359 { id: "high", label: "High", isDefault: true },
360 { id: "xhigh", label: "Extra High (capped to High)" },
361 { id: "max", label: "Max" },
362 { id: "ultracode", label: "Ultracode" },
363 { id: "ultrathink", label: "Ultrathink" },
364 ],
365 ]);
366 // An effortMap that sends Extra High as max stays unmarked under an xhigh cap: the CLI runs
367 // it at xhigh, which is what the label says.
368 assert.deepStrictEqual(
369 options("xhigh").map((model) => model?.map((option) => option.label)),
370 [
371 ["Medium", "High", "Extra High", "Max (capped to Extra High)", "Ultracode", "Ultrathink"],
372 ["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"],
373 ],
374 );
375
376 // A per-model cap applies without a top-level one and leaves other models uncapped.
377 assert.deepStrictEqual(
378 capClaudeCatalogEffort(catalog, models, [
379 { modelMaxEffortLevels: new Map([["claude-plain", "high"]]) },
380 ]).map((model) => {
381 const effort = model.capabilities?.optionDescriptors?.[0];
382 return effort?.type === "select" ? effort.options.map((option) => option.label) : [];
383 }),
384 [
385 [
386 "Medium",
387 "High",
388 "Extra High (capped to High)",
389 "Max (capped to High)",
390 "Ultracode (capped to High)",
391 "Ultrathink",
392 ],
393 ["Medium", "High", "Extra High", "Max", "Ultracode", "Ultrathink"],
394 ],
395 );
396
397 // Without an Extra High option (Opus 4.6), the CLI clamps an xhigh cap down to High.
398 const noXhigh = {
399 slug: "claude-no-xhigh",
400 name: "claude-no-xhigh",
401 isCustom: false,
402 capabilities: {
403 optionDescriptors: [
404 {
405 id: "effort",
406 label: "Reasoning",
407 type: "select" as const,
408 options: [
409 { id: "low", label: "Low" },
410 { id: "medium", label: "Medium" },
411 { id: "high", label: "High", isDefault: true },
412 { id: "max", label: "Max" },
413 ],
414 },
415 ],
416 },
417 };
418 const [capped] = capClaudeCatalogEffort(
419 { models: [] },
420 [noXhigh],
421 [{ maxEffortLevel: "xhigh" }],
422 );
423 const cappedEffort = capped?.capabilities?.optionDescriptors?.[0];
424 assert.deepStrictEqual(
425 cappedEffort?.type === "select" ? cappedEffort.options.map((option) => option.label) : [],
426 ["Low", "Medium", "High", "Max (capped to High)"],
427 );
428 });
429
430 it("leaves custom model effort labels alone under a cap", () => {
431 const custom = {
432 slug: "claude-custom-copy",
433 name: "Copy",
434 isCustom: true,
435 capabilities: {
436 optionDescriptors: [
437 {
438 id: "effort",
439 label: "Reasoning",
440 type: "select" as const,
441 options: [
442 { id: "high", label: "High", isDefault: true },
443 { id: "max", label: "Max (capped to High)" },
444 ],
445 },
446 ],
447 },
448 };
449
450 assert.deepStrictEqual(
451 capClaudeCatalogEffort({ models: [] }, [custom], [{ maxEffortLevel: "high" }]),
452 [custom],
453 );
454 });
455
456 it("resolves maxEffortLevel per settings source and per model key like Claude Code", () => {
457 const opus = {
458 slug: "claude-opus-5-5",
459 name: "Opus 5.5",
460 isCustom: false,
461 aliases: ["opus", "opus-5.5", "claude-opus-5.5"],
462 capabilities: {
463 optionDescriptors: [
464 {
465 id: "effort",
466 label: "Reasoning",
467 type: "select" as const,
468 options: [
469 { id: "low", label: "Low" },
470 { id: "medium", label: "Medium" },
471 { id: "high", label: "High", isDefault: true },
472 { id: "max", label: "Max" },
473 ],
474 },
475 ],
476 },
477 };
478 const catalog: ClaudeModelCatalog = {
479 models: [
480 { model: opus, runtime: {}, compatibility: {} },
481 {
482 model: {
483 slug: "claude-opus-5",
484 name: "Opus 5",
485 isCustom: false,
486 capabilities: { optionDescriptors: [] },
487 },
488 runtime: {},
489 compatibility: {},
490 },
491 ],
492 };
493 const maxLabel = (sources: Parameters<typeof capClaudeCatalogEffort>[2]) => {
494 const effort = capClaudeCatalogEffort(catalog, [opus], sources)[0]?.capabilities
495 ?.optionDescriptors?.[0];
496 return effort?.type === "select" ? effort.options.at(-1)?.label : undefined;
497 };
498 const perModel = (entries: Record<string, "low" | "medium" | "high" | "max">) => ({
499 modelMaxEffortLevels: new Map(Object.entries(entries)),
500 });
501
502 // Claude Code matches family aliases and [1m] spellings case-insensitively.
503 assert.strictEqual(maxLabel([perModel({ Opus: "medium" })]), "Max (capped to Medium)");
504 assert.strictEqual(
505 maxLabel([perModel({ "claude-opus-5-5[1M]": "medium" })]),
506 "Max (capped to Medium)",
507 );
508 assert.strictEqual(maxLabel([perModel({ " OPUS[1m] ": "medium" })]), "Max (capped to Medium)");
509 // Other keys name the most specific model id they contain, so dated and provider-prefixed ids
510 // match while the catalog's dotted aliases and a less specific model id do not.
511 for (const key of [
512 "claude-opus-5-5-20260101",
513 "CLAUDE-OPUS-5-5-20260101[1m]",
514 "us.anthropic.claude-opus-5-5-v1:0",
515 "claude-opus-5-5-latest",
516 ]) {
517 assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max (capped to Medium)", key);
518 }
519 for (const key of ["claude-opus-5.5", "opus-5.5", "opus-5-5", "opus [1m]", "claude-opus-5"]) {
520 assert.strictEqual(maxLabel([perModel({ [key]: "medium" })]), "Max", key);
521 }
522 // Keys naming the same model keep the lowest cap, and a `max` key does not exempt it.
523 assert.strictEqual(
524 maxLabel([perModel({ opus: "max", "claude-opus-5-5": "low" })]),
525 "Max (capped to Low)",
526 );
527 // A per-model `max` only exempts the model from its own source's top-level cap.
528 assert.strictEqual(
529 maxLabel([{ maxEffortLevel: "medium", ...perModel({ opus: "max" }) }]),
530 "Max",
531 );
532 assert.strictEqual(
533 maxLabel([perModel({ opus: "max" }), { maxEffortLevel: "high" }]),
534 "Max (capped to High)",
535 );
536 assert.strictEqual(
537 maxLabel([{ maxEffortLevel: "medium" }, perModel({ opus: "high" })]),
538 "Max (capped to Medium)",
539 );
540 });
541 });