@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 3,449 lines 149 KB
Raw
1 #!/usr/bin/env node
2 // @effect-diagnostics nodeBuiltinImport:off
3 /**
4 * Replays small fake threads on a Claude or Codex model to check T3's MCP tool use. A trimmed
5 * harness capture supplies the prompt and native tools; a cheap model plays the shell. See
6 * README.md for how to run it and add cases.
7 */
8 import * as NodeChildProcess from "node:child_process";
9 import * as NodeCrypto from "node:crypto";
10 import * as NodeUtil from "node:util";
11 import * as NodeVM from "node:vm";
12 import * as NodeZlib from "node:zlib";
13 import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
14 import * as PlatformNodeCrypto from "@effect/platform-node/NodeCrypto";
15 import * as NodeServices from "@effect/platform-node/NodeServices";
16 import {
17 ComposerContextRecord,
18 DeviceOperationError,
19 DeviceSummary,
20 OrchestratorMcpTaskRole,
21 PreviewAutomationError,
22 PreviewAutomationUnavailableError,
23 ProviderInstanceId,
24 } from "@t3tools/contracts";
25 import * as Cause from "effect/Cause";
26 import * as Context from "effect/Context";
27 import * as Config from "effect/Config";
28 import * as Effect from "effect/Effect";
29 import * as FileSystem from "effect/FileSystem";
30 import * as Layer from "effect/Layer";
31 import * as Option from "effect/Option";
32 import * as Path from "effect/Path";
33 import * as Schedule from "effect/Schedule";
34 import * as Schema from "effect/Schema";
35 import * as Semaphore from "effect/Semaphore";
36 import { Tool } from "effect/ai";
37 import * as McpSchema from "effect/ai/McpSchema";
38 import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http";
39
40 import { taskPrompt } from "../../apps/server/src/mcp/delegatedTaskModes.ts";
41 import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts";
42 import { advertisedJsonSchema } from "../../apps/server/src/mcp/advertisedJsonSchema.ts";
43 import * as Contracts from "@t3tools/contracts";
44 import {
45 EnvironmentId,
46 ORCHESTRATOR_MCP_RUNNING_TASK_NEXT,
47 PullRequestState,
48 ThreadId,
49 } from "@t3tools/contracts";
50 import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences";
51 import { THEME_GUIDE } from "@t3tools/shared/themeCss";
52 import { formatThreadLink } from "@t3tools/shared/threadLinks";
53 import { buildCodexAdditionalContext } from "../../apps/server/src/provider/CodexDeveloperInstructions.ts";
54 import { PULL_REQUEST_LINKING_INSTRUCTIONS } from "@t3tools/provider-core/server/runtimeInstructions";
55 import { makeSideChatContext } from "../../apps/server/src/orchestration-v2/ContextHandoffService.ts";
56 import { withInlineContext } from "../../apps/server/src/orchestration-v2/ContextHandoffDelivery.ts";
57 import { AttachmentToolkit } from "../../apps/server/src/mcp/toolkits/attachment/tools.ts";
58 import { DeviceToolkit } from "../../apps/server/src/mcp/toolkits/device/tools.ts";
59 import { EnvironmentToolkit } from "../../apps/server/src/mcp/toolkits/environment/tools.ts";
60 import { HtmlToolkit } from "../../apps/server/src/mcp/toolkits/html/tools.ts";
61 import { IssuesToolkit } from "../../apps/server/src/mcp/toolkits/issues/tools.ts";
62 import { MessageToolkit } from "../../apps/server/src/mcp/toolkits/message/tools.ts";
63 import { OrchestratorToolkit } from "../../apps/server/src/mcp/toolkits/orchestrator/tools.ts";
64 import { PreviewToolkit } from "../../apps/server/src/mcp/toolkits/preview/tools.ts";
65 import { PreviewControlsToolkit } from "../../apps/server/src/mcp/toolkits/previewControls/tools.ts";
66 import { ProjectToolkit } from "../../apps/server/src/mcp/toolkits/project/tools.ts";
67 import {
68 CHILD_PULL_REQUEST_REPORT_NEXT,
69 PullRequestsToolkit,
70 } from "../../apps/server/src/mcp/toolkits/pullRequests/tools.ts";
71 import { ThemeToolkit } from "../../apps/server/src/mcp/toolkits/theme/tools.ts";
72 import { ThreadToolkit } from "../../apps/server/src/mcp/toolkits/thread/tools.ts";
73 import { WorktreeToolkit } from "../../apps/server/src/mcp/toolkits/worktree/tools.ts";
74 import {
75 CLAUDE_PROVIDER,
76 makeClaudeQueryOptions,
77 } from "../../apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts";
78 import { proxyConfig } from "../shared/proxy.ts";
79 import { changeRequestsCreatedBy } from "../../apps/server/src/orchestration-v2/PullRequestSyncReactor.ts";
80 import { type BackgroundWait, makeBackgroundWait } from "./backgroundWait.ts";
81 import { DeviceExpect, gradeDevice } from "./device-grading.ts";
82 import { FileLinksExpect, gradeFileLinks } from "./file-link-grading.ts";
83 import {
84 AGENT_DEVICE_SHELL,
85 deviceFailure,
86 type DeviceTools,
87 isDeviceTool,
88 launcherHelp,
89 makeDeviceTools,
90 } from "./devices.ts";
91 import {
92 argvForm,
93 gradeSecret,
94 scriptsBefore,
95 UNKNOWN_SCRIPT,
96 scriptSources,
97 SecretExpect,
98 secretDiagnostics,
99 } from "./secret-grading.ts";
100 /** Agent models by id, with the display name a harness gives them. */
101 const MODELS = {
102 "claude-sonnet-5-5": "Sonnet 5.5",
103 "claude-opus-5-5": "Opus 5.5",
104 "gpt-6.1-sol": "GPT-6.1 Sol",
105 "gpt-6-astra": "GPT-6 Astra",
106 "gpt-6-luna": "GPT-6 Luna",
107 } as const;
108 type Model = keyof typeof MODELS;
109 /** GPT models replay a Codex capture through CLIProxyAPI, the rest a Claude Code capture. */
110 const isCodex = (model: Model): model is Extract<Model, `gpt-${string}`> =>
111 model.startsWith("gpt-");
112 const EFFORTS = ["low", "medium", "high", "xhigh", "max"] as const;
113 // USD per million tokens. Cache writes use the 5-minute rate.
114 export const PRICES = {
115 "claude-sonnet-5-5": { input: 2, output: 10, cacheWrite: 2.5, cacheRead: 0.2 },
116 "claude-haiku-5-5": { input: 0.1, output: 0.5, cacheWrite: 0.125, cacheRead: 0.01 },
117 "claude-opus-5-5": { input: 4, output: 20, cacheWrite: 5, cacheRead: 0.2 },
118 };
119 type PricedModel = keyof typeof PRICES;
120 /** GPT models that can play the shell through CLIProxyAPI, when no Anthropic key is at hand. */
121 const PROXY_SHELL_MODELS = ["gpt-6-luna", "gpt-6.1-terra"] as const;
122 type ShellModel = PricedModel | (typeof PROXY_SHELL_MODELS)[number];
123 const isPricedModel = (model: ShellModel): model is PricedModel => model in PRICES;
124 const T3_PREFIX = "mcp__t3-code__";
125 const RUNTIME_MARKER = "<runtime_info>";
126 const TOOL_LOADED = "Tool loaded.";
127
128 class EvalToolCallError extends Schema.TaggedError<EvalToolCallError>()("EvalToolCallError", {
129 tool: Schema.String,
130 cause: Schema.Defect(),
131 }) {
132 override get message(): string {
133 return `The ${this.tool} tool call failed.`;
134 }
135 }
136 const DEFERRED_LIST =
137 /(The following deferred tools are now available via ToolSearch\.[^\n]*\n)((?:[^\n]+\n)*)/;
138 const MODEL_LINE = /You are powered by the model named .+?\. The exact model ID is [\w.-]+\./;
139 // API keys can be billed differently for requests that open with Claude Code's identity line,
140 // so every copy of it becomes a neutral line. The rest of the captured prompt is unchanged.
141 const IDENTITY =
142 /You are Claude Code, Anthropic's official CLI for Claude(?:, running within the Claude Agent SDK)?\./g;
143 const NEUTRAL_IDENTITY = "You are an AI coding agent running in the user's terminal.";
144
145 type Block = { readonly type: string; readonly [key: string]: unknown };
146 type ApiMessage = { readonly role: "user" | "assistant"; readonly content: ReadonlyArray<Block> };
147 type ApiTool = { readonly name: string; readonly [key: string]: unknown };
148 type Usage = Record<keyof (typeof PRICES)[PricedModel], number>;
149 type Call = {
150 readonly step: number;
151 readonly tool: string;
152 readonly input: unknown;
153 /** The tool_use id, when the call was a tool use of its own. */
154 readonly id?: string;
155 /** Once answered: whether the agent saw an error, and the result text it saw. */
156 failed?: boolean;
157 output?: string;
158 };
159 export type SampleRun = {
160 readonly calls: ReadonlyArray<Call>;
161 text: string;
162 stopReason: string;
163 /** The agent's final answer: its last response's text, if it ended its turn. */
164 readonly finalText?: string;
165 /** PRs linked to the thread when the sample ends, by the agent or by T3. */
166 readonly linked?: ReadonlyArray<string>;
167 /** The thread's working directory, which chat resolves relative file pills against. */
168 readonly cwd?: string;
169 /** The `agent-device` commands T3's device_open returned, prior turns included. */
170 readonly launchers?: ReadonlyArray<string>;
171 };
172
173 const Rest = [Schema.Record(Schema.String, Schema.Unknown)] as const;
174
175 const PriorToolCall = Schema.Struct({
176 tool: Schema.String,
177 input: Schema.Unknown,
178 /**
179 * For a device tool or t3_background_wait, `"default"` runs the call on the world's devices or
180 * background tasks for T3's result.
181 */
182 result: Schema.Unknown,
183 /** Repository-relative file whose current contents supply this tool result. */
184 resultFile: Schema.optional(Schema.String),
185 });
186
187 /**
188 * One expected call. `args` match partially: `~text` is a case-insensitive substring, `re:` a
189 * case-sensitive regex, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths.
190 */
191 const McpToolUseCall = Schema.Struct({
192 tool: Schema.String,
193 args: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
194 });
195 const McpToolUseCallWithAlternatives = Schema.Struct({
196 ...McpToolUseCall.fields,
197 argsAnyOf: Schema.optional(Schema.Array(Schema.Record(Schema.String, Schema.Unknown))),
198 });
199 const McpToolUseExpectedCall = Schema.Struct({
200 ...McpToolUseCallWithAlternatives.fields,
201 success: Schema.optional(Schema.Literal(true)),
202 });
203 type McpToolUseCall = typeof McpToolUseExpectedCall.Type;
204
205 /** Grading rules. Tool names are bare: `delegate_task`, `Bash`, `Agent`. */
206 const McpToolUseCallsExpect = Schema.Struct({
207 /** Each entry needs one matching call. */
208 calls: Schema.optional(Schema.Array(McpToolUseExpectedCall)),
209 /** Each entry matches a different call, and no call matches two entries; tool names are exact. */
210 distinctCalls: Schema.optional(Schema.Array(McpToolUseCall)),
211 minCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
212 maxCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
213 });
214
215 /** One `anyOf` alternative: call checks, and text checks that must hold on the same path. */
216 const McpToolUseAlternative = Schema.Struct({
217 ...McpToolUseCallsExpect.fields,
218 textMust: Schema.optional(Schema.Array(Schema.String)),
219 finalTextMust: Schema.optional(Schema.Array(Schema.String)),
220 });
221
222 const McpToolUseExpect = Schema.Struct({
223 ...McpToolUseCallsExpect.fields,
224 /** Passes when one alternative passes every one of its checks. */
225 anyOf: Schema.optional(Schema.Array(McpToolUseAlternative)),
226 /** No call may match any entry. */
227 callsMustNot: Schema.optional(Schema.Array(McpToolUseCallWithAlternatives)),
228 notCalls: Schema.optional(Schema.Array(Schema.String)),
229 /** First calls of these tools happen in this order. */
230 order: Schema.optional(Schema.Array(Schema.String)),
231 /**
232 * One `args`-style value per new top-level thread the agent asks for, by `t3_thread_launch` or a
233 * `create_threads` entry. Each matches a different thread's request; no other thread is asked for.
234 */
235 newThreads: Schema.optional(Schema.Array(Schema.Unknown)),
236 /** The last step with tool calls includes this tool, then the turn ends. */
237 endsTurnAfter: Schema.optional(Schema.String),
238 /**
239 * Regexes that no shell command may match. A command also carries the source of the scripts it
240 * runs that the agent wrote earlier in the sample.
241 */
242 shellMustNot: Schema.optional(Schema.Array(Schema.String)),
243 /** Regexes that some shell command must match, each. */
244 shellMust: Schema.optional(Schema.Array(Schema.String)),
245 /** Case-sensitive regexes on the agent's visible text. */
246 textMust: Schema.optional(Schema.Array(Schema.String)),
247 textMustNot: Schema.optional(Schema.Array(Schema.String)),
248 /** Case-sensitive regexes on the final answer: the last response's text, once the turn ended. */
249 finalTextMust: Schema.optional(Schema.Array(Schema.String)),
250 /** Each entry matches a PR URL linked at the end, as `args` values match. */
251 linked: Schema.optional(Schema.Array(Schema.String)),
252 /** The PR numbers linked at the end, exactly. */
253 linkedNumbers: Schema.optional(Schema.Array(Schema.Number)),
254 /** Device manager checks: child briefs, simulator UI over the shell, failures, recordings. */
255 device: Schema.optional(DeviceExpect),
256 /**
257 * Secret checks: chat asks in text and question tools, where fixture values went, and how the
258 * files request_secret returned were read. See `SecretExpect` in secret-grading.ts.
259 */
260 secret: Schema.optional(SecretExpect),
261 /** File pills in the final answer resolve against the thread cwd to `world.files` entries. */
262 fileLinks: Schema.optional(FileLinksExpect),
263 });
264
265 /**
266 * The agent's machine, as the shell simulator and the gitStatus reminder see it. Only these keys
267 * are typed; others (files, diff, prs, pages) are free-form facts for the simulator.
268 */
269 const McpToolUseWorld = Schema.StructWithRest(
270 Schema.Struct({
271 /** GitHub `owner/name`. Defaults to `acme/app`. */
272 repo: Schema.optional(Schema.String),
273 /** `origin`'s host and path, such as `git.cafe/acme/app`. Defaults to `github.com/<repo>`. */
274 remote: Schema.optional(Schema.String),
275 /** The branch has commits but no upstream: T3's pull_request_create refuses until a push. */
276 unpushed: Schema.optional(Schema.Boolean),
277 branch: Schema.optional(Schema.String),
278 /** `git status --short` lines. */
279 status: Schema.optional(Schema.String),
280 /** Newest first. On a branch other than main, these are the commits main lacks. */
281 commits: Schema.optional(
282 Schema.Array(
283 Schema.StructWithRest(Schema.Struct({ sha: Schema.String, subject: Schema.String }), Rest),
284 ),
285 ),
286 /** Simulators and emulators. With them, T3's device tools answer from this world. */
287 devices: Schema.optional(Schema.Array(DeviceSummary)),
288 /** Labels of the SSH device hosts in `devices`, by id. The local host always exists. */
289 deviceHosts: Schema.optional(Schema.Record(Schema.String, Schema.String)),
290 /**
291 * agent-device sessions that prior turns started, by device id, with the app `open` put in
292 * each. Otherwise a device's session starts at its launcher's first `open` in the sample.
293 */
294 agentDeviceSessions: Schema.optional(
295 Schema.Record(Schema.String, Schema.Struct({ app: Schema.optional(Schema.String) })),
296 ),
297 /**
298 * Hosts whose hub fails to list devices, by id, with the error, such as F1's
299 * `{"_tag": "DeviceOperationError", "operation": "list", "reason": "request_failed", "cause": null}`.
300 * T3 marks each one failed on every call and lists none of its devices.
301 */
302 deviceDiscoveryFailures: Schema.optional(Schema.Record(Schema.String, DeviceOperationError)),
303 /** false: agent device access is off. T3 denies every device call and drops its device text. */
304 deviceAgentAccess: Schema.optional(Schema.Boolean),
305 /** false: agent browser access is off. T3 denies every preview call and drops its browser text. */
306 browserAgentAccess: Schema.optional(Schema.Boolean),
307 /** Every capture reaches T3 with adb screencap's warning ahead of the PNG: T3 reports 0×0. */
308 deviceInvalidCapture: Schema.optional(Schema.Boolean),
309 /** Devices another thread's agent-device session holds, by device id: that thread's id. */
310 deviceOwners: Schema.optional(Schema.Record(Schema.String, Schema.String)),
311 /**
312 * Task IDs of the background commands and Monitors still running on the agent's Claude
313 * session. With them, T3's own t3_background_wait handler answers from this world.
314 */
315 backgroundTasks: Schema.optional(Schema.Array(Schema.String)),
316 /** The world's `prs`, bottom first, form one native stack with this number. */
317 stack: Schema.optional(Schema.Number),
318 /** PRs already linked to the thread, with the state T3 last saw. */
319 linkedPullRequests: Schema.optional(
320 Schema.Array(
321 Schema.Struct({ number: Schema.Number, state: PullRequestState, title: Schema.String }),
322 ),
323 ),
324 }),
325 Rest,
326 );
327
328 /** One fake thread, one JSON object per line in a case file. */
329 export const McpToolUseCase = Schema.Struct({
330 id: Schema.String,
331 area: Schema.String,
332 kind: Schema.Literals(["bad", "guard", "control"]),
333 pattern: Schema.String,
334 experience: Schema.String,
335 rule: Schema.optional(Schema.String),
336 privacy: Schema.optional(Schema.Literals(["commit-safe", "private"])),
337 setup: Schema.Struct({
338 role: Schema.Literals(["top", "delegated-child", "native-subagent"]),
339 /** Runs only on this harness, for cases whose right answer depends on it. */
340 harness: Schema.optional(Schema.Literals(["claude", "codex"])),
341 /** The Linked environments beta is on, so T3 lists its link tools. */
342 linkedEnvironments: Schema.optional(Schema.Boolean),
343 /** false: the project is on no forge T3 reads, so T3 hides its forge tools. */
344 forge: Schema.optional(Schema.Boolean),
345 /** delegate_task `role` for a delegated child. T3 builds the child's first message from it. */
346 delegateRole: Schema.optional(OrchestratorMcpTaskRole),
347 context: Schema.optional(Schema.String),
348 priorTurns: Schema.optional(
349 Schema.Array(
350 Schema.Struct({
351 role: Schema.Literals(["user", "assistant"]),
352 text: Schema.optional(Schema.String),
353 toolCalls: Schema.optional(Schema.Array(PriorToolCall)),
354 }),
355 ),
356 ),
357 world: Schema.optional(McpToolUseWorld),
358 /**
359 * Canned tool results by bare name, T3 or native. `"default"` uses the runner's result. For a
360 * device tool, a T3 `DeviceError` (`{"_tag": ..., "cause": null}`) fails it inside T3; for a
361 * preview tool, a `PreviewAutomationError` fails it with that error's message. A
362 * `{"$sequence": [...]}` mock answers its nth call with the nth result, then repeats the last.
363 */
364 mocks: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
365 /** Regexes on one part of a shell command. A match wins over the shell simulator. */
366 shellMocks: Schema.optional(
367 Schema.Array(Schema.Struct({ match: Schema.String, output: Schema.String })),
368 ),
369 /** The thread is a side chat's first turn: T3 puts the parent's digest ahead of `prompt`. */
370 sideChat: Schema.optional(
371 Schema.Struct({ parentThreadId: Schema.String, digest: Schema.Array(Schema.String) }),
372 ),
373 }),
374 /** Empty to resume a top-level thread without side-chat context after its last tool result. */
375 prompt: Schema.String,
376 /** Records for the context chips `prompt` links, projected as T3 sends them. */
377 promptContext: Schema.optional(Schema.Struct({ records: Schema.Array(ComposerContextRecord) })),
378 expect: McpToolUseExpect,
379 /**
380 * The checks for `--arm raw`, when its right outcome differs, as when only T3 can watch a PR.
381 * Otherwise both arms are graded on `expect`.
382 */
383 expectRaw: Schema.optional(McpToolUseExpect),
384 /** Model calls this case may need. Overrides `--max-steps`. */
385 maxSteps: Schema.optional(Schema.Number),
386 source: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
387 notes: Schema.optional(Schema.String),
388 });
389 export type McpToolUseCase = typeof McpToolUseCase.Type;
390 const AnyBlock = Schema.StructWithRest(
391 Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }),
392 Rest,
393 );
394 const Content = Schema.Union([Schema.String, Schema.Array(AnyBlock)]);
395 const MessagesRequest = Schema.Struct({
396 system: Schema.optional(Content),
397 messages: Schema.Array(Schema.Struct({ role: Schema.String, content: Content })),
398 tools: Schema.Array(Schema.StructWithRest(Schema.Struct({ name: Schema.String }), Rest)),
399 });
400 // A CLIProxyAPI capture keeps the Anthropic body at request.provider_native.
401 const Capture = Schema.Struct({ request: Schema.Struct({ provider_native: MessagesRequest }) });
402 const ResponsesRequest = Schema.StructWithRest(
403 Schema.Struct({
404 input: Schema.Array(AnyBlock),
405 reasoning: Schema.StructWithRest(Schema.Struct({ effort: Schema.String }), Rest),
406 }),
407 Rest,
408 );
409 const CodexCapture = Schema.Struct({
410 request: Schema.Struct({ provider_native: ResponsesRequest }),
411 });
412 const ToolUse = Schema.Struct({
413 type: Schema.Literal("tool_use"),
414 id: Schema.String,
415 name: Schema.String,
416 input: Schema.Unknown,
417 });
418 const isToolUse = Schema.is(ToolUse);
419 const isText = Schema.is(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }));
420 const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
421 typeof value === "object" && value !== null && !Array.isArray(value);
422 const Count = Schema.optional(Schema.NullOr(Schema.Number));
423 const ApiResponse = Schema.Struct({
424 content: Schema.Array(AnyBlock),
425 stop_reason: Schema.NullOr(Schema.String),
426 usage: Schema.Struct({
427 input_tokens: Schema.Number,
428 output_tokens: Schema.Number,
429 cache_creation_input_tokens: Count,
430 cache_read_input_tokens: Count,
431 }),
432 });
433
434 const emptyUsage = (): Usage => ({ input: 0, output: 0, cacheWrite: 0, cacheRead: 0 });
435 const addUsage = (total: Usage, usage: typeof ApiResponse.Type.usage) => {
436 total.input += usage.input_tokens;
437 total.output += usage.output_tokens;
438 total.cacheWrite += usage.cache_creation_input_tokens ?? 0;
439 total.cacheRead += usage.cache_read_input_tokens ?? 0;
440 };
441
442 const text = (value: string): Block => ({ type: "text", text: value });
443 const reminder = (value: string) => text(`<system-reminder>\n${value}\n</system-reminder>`);
444 const valueText = (value: unknown) =>
445 typeof value === "string" ? value : (JSON.stringify(value) ?? "");
446 const toolResult = (id: string, content: unknown): Block => ({
447 type: "tool_result",
448 tool_use_id: id,
449 content: valueText(content),
450 });
451 const texts = (content: typeof Content.Type) =>
452 typeof content === "string" ? [content] : content.flatMap((block) => block.text ?? []);
453 const isT3Name = (name: string) => name.startsWith(T3_PREFIX) || name.startsWith("mcp__t3_code__");
454 const hasTool = (tools: ReadonlyArray<ApiTool>, name: string) =>
455 tools.some((tool) => tool.name === name);
456
457 /** Grading name: T3 tools lose their MCP prefix, native tools keep the part after the last `_`. */
458 const CODEX_NAMES: Readonly<Record<string, string>> = {
459 exec_command: "Bash",
460 shell_command: "Bash",
461 shell: "Bash",
462 spawn_agent: "Agent",
463 apply_patch: "Edit",
464 request_user_input: "AskUserQuestion",
465 request_user_input_async: "AskUserQuestion",
466 write_stdin: "write_stdin",
467 web__run: "WebFetch",
468 };
469 export const normalizeToolName = (name: string) => {
470 const bare = name.slice(name.lastIndexOf(".") + 1);
471 return isT3Name(bare)
472 ? bare.slice(T3_PREFIX.length)
473 : (CODEX_NAMES[bare] ?? bare.slice(bare.lastIndexOf("_") + 1));
474 };
475 export const normalizeToolInput = (name: string, input: unknown) => {
476 const decode = t3InputDecoders.get(normalizeToolName(name));
477 if (decode !== undefined) return Option.getOrElse(decode(input), () => input);
478 return normalizeToolName(name) === "Bash" && isRecord(input)
479 ? { ...input, command: input.command ?? input.cmd }
480 : typeof input === "string" && normalizeToolName(name) === "Edit"
481 ? { patch: input }
482 : input;
483 };
484
485 // McpHttpServer advertises each tool's JSON schema, and Claude Code passes the MCP name,
486 // description, and schema through unchanged.
487 const t3SourceTools: ReadonlyArray<Tool.Any> = [
488 OrchestratorToolkit,
489 ThreadToolkit,
490 AttachmentToolkit,
491 ProjectToolkit,
492 EnvironmentToolkit,
493 ThemeToolkit,
494 PreviewControlsToolkit,
495 WorktreeToolkit,
496 PullRequestsToolkit,
497 PreviewToolkit,
498 IssuesToolkit,
499 DeviceToolkit,
500 HtmlToolkit,
501 MessageToolkit,
502 ].flatMap((toolkit): ReadonlyArray<Tool.Any> => Object.values(toolkit.tools));
503 // McpHttpServer decodes arguments with each tool's JSON codec, which drops undeclared keys.
504 const t3InputDecoders = new Map(
505 t3SourceTools.map((tool) => [
506 tool.name,
507 Schema.decodeUnknownOption(
508 Schema.toCodecJson(tool.parametersSchema as Schema.Codec<unknown, unknown>),
509 ),
510 ]),
511 );
512 const t3Tools: ReadonlyArray<ApiTool> = t3SourceTools.map((tool) => ({
513 name: `${T3_PREFIX}${tool.name}`,
514 description: Tool.getDescription(tool) ?? "",
515 input_schema: advertisedJsonSchema(tool),
516 }));
517 // T3 gates these on the `preview` capability, which agent browser access decides.
518 const PREVIEW_TOOLS = new Set(
519 [PreviewToolkit, PreviewControlsToolkit].flatMap((toolkit) =>
520 Object.values(toolkit.tools).map((tool: Tool.Any) => tool.name),
521 ),
522 );
523 const decodePreviewError = Schema.decodeUnknownOption(PreviewAutomationError);
524 const previewFailure = (mock: unknown) => Option.getOrUndefined(decodePreviewError(mock));
525 const enabledWhen = (predicate: () => boolean) =>
526 new Set(
527 t3SourceTools
528 .filter(
529 (tool) => Context.getOrUndefined(tool.annotations, McpSchema.EnabledWhen) === predicate,
530 )
531 .map((tool) => `${T3_PREFIX}${tool.name}`),
532 );
533 // Claude Code loads a tool whose `_meta` sets anthropic/alwaysLoad up front instead of deferring it.
534 const alwaysLoaded = new Set(
535 t3SourceTools
536 .filter((tool) => {
537 const meta = Context.getOrUndefined(tool.annotations, Tool.Meta);
538 return meta !== undefined && meta["anthropic/alwaysLoad"] === true;
539 })
540 .map((tool) => `${T3_PREFIX}${tool.name}`),
541 );
542 // Refs whose request_secret takes `value` save a secret the user already pasted, with no card.
543 export const savesPastedSecrets = t3Tools.some(
544 (tool) =>
545 tool.name === `${T3_PREFIX}request_secret` &&
546 isRecord(tool.input_schema) &&
547 isRecord(tool.input_schema.properties) &&
548 "value" in tool.input_schema.properties,
549 );
550 /** A request_secret call that hands T3 a pasted secret, on a ref that takes one. */
551 const pastedSecret = (name: string, input: Readonly<Record<string, unknown>>) =>
552 savesPastedSecrets && name === "request_secret" && typeof input.value === "string";
553 // T3 does not offer these to a delegated child, so its harness never lists them.
554 const hiddenFromSubagents = enabledWhen(McpInvocationContext.hiddenFromSubagents);
555 // The Linked environments beta starts off, and T3 lists its tools only while it is on.
556 const hiddenWhileLinksOff = enabledWhen(McpInvocationContext.hiddenWhileLinksOff);
557 // T3 lists PR and CI tools only to a thread whose project is on a forge it reads.
558 const hiddenWithoutForge = enabledWhen(McpInvocationContext.hiddenWithoutForge);
559 /**
560 * `tools` is T3 as shipped. `raw` takes away T3's PR, CI and issue tools and their instructions,
561 * so the agent works the forge with CLIs alone, to measure what the tools save.
562 */
563 export const ARMS = ["tools", "raw"] as const;
564 export type Arm = (typeof ARMS)[number];
565 const forgeToolNames = new Set(
566 [PullRequestsToolkit, IssuesToolkit].flatMap((toolkit) =>
567 Object.values(toolkit.tools).map((tool: Tool.Any) => `${T3_PREFIX}${tool.name}`),
568 ),
569 );
570 /** T3's text for an arm: `raw` drops the PR linking block that names T3's PR tools. */
571 const forArm = (value: string, arm: Arm | undefined) =>
572 arm === "raw" ? value.replace(`\n\n${PULL_REQUEST_LINKING_INSTRUCTIONS}`, "") : value;
573 const visibleT3Tools = (options: {
574 readonly subagent?: boolean;
575 readonly links?: boolean;
576 readonly forge?: boolean;
577 readonly arm?: Arm;
578 }) =>
579 t3Tools.filter(
580 (tool) =>
581 !(options.arm === "raw" && forgeToolNames.has(tool.name)) &&
582 (options.links === true || !hiddenWhileLinksOff.has(tool.name)) &&
583 (options.forge !== false || !hiddenWithoutForge.has(tool.name)) &&
584 !(options.subagent === true && hiddenFromSubagents.has(tool.name)),
585 );
586
587 /**
588 * T3's Claude Code options for an eval thread: its system prompt append and its hooks. `device` and
589 * `browser` say whether the thread holds the `device` and `preview` capabilities, as agent device
590 * and browser access decide.
591 */
592 const claudeQueryOptions = (model: Model, device = true, browser = true) => {
593 // Refs whose text follows a capability read its flag; older ones ignore it.
594 const input: Parameters<typeof makeClaudeQueryOptions>[0] & {
595 readonly deviceToolsAvailable?: boolean;
596 readonly browserToolsAvailable?: boolean;
597 } = {
598 modelSelection: { instanceId: ProviderInstanceId.make(CLAUDE_PROVIDER), model },
599 nativeThreadId: "mcp-tool-use-eval",
600 resume: false,
601 cwd: null,
602 mcpServers: { "t3-code": { type: "http", url: "http://127.0.0.1/mcp" } },
603 deviceToolsAvailable: device,
604 browserToolsAvailable: browser,
605 };
606 return makeClaudeQueryOptions(input);
607 };
608
609 /** Text T3 appends to Claude Code's system prompt. Delegated children get the same text. */
610 export const t3SystemAppend = (model: Model, device = true, browser = true) => {
611 const { systemPrompt } = claudeQueryOptions(model, device, browser);
612 if (isRecord(systemPrompt) && systemPrompt.type === "preset" && systemPrompt.append) {
613 return systemPrompt.append;
614 }
615 throw new Error("makeClaudeQueryOptions no longer appends T3 text to a preset prompt.");
616 };
617
618 /**
619 * The context T3's SubagentStart hooks give a native subagent, which gets none of the append.
620 * Empty when T3 registers no such hook.
621 */
622 export const t3SubagentContext = async (model: Model, device = true, browser = true) => {
623 const hooks = (claudeQueryOptions(model, device, browser).hooks?.SubagentStart ?? []).flatMap(
624 (m) => m.hooks,
625 );
626 const input = {
627 hook_event_name: "SubagentStart" as const,
628 session_id: "mcp-tool-use-eval",
629 transcript_path: "",
630 cwd: "",
631 agent_id: "mcp-tool-use-eval",
632 agent_type: "general-purpose",
633 };
634 const outputs = await Promise.all(
635 hooks.map((hook) => hook(input, undefined, { signal: AbortSignal.timeout(10_000) })),
636 );
637 return outputs
638 .flatMap((output) =>
639 "hookSpecificOutput" in output && output.hookSpecificOutput?.hookEventName === "SubagentStart"
640 ? [output.hookSpecificOutput.additionalContext ?? ""]
641 : [],
642 )
643 .filter(Boolean)
644 .join("\n\n");
645 };
646
647 /**
648 * Size of the context T3 adds: the system text every Claude thread gets, and the tool
649 * descriptions and schemas (loaded on demand here, sent up front by harnesses without
650 * deferred tools). Tokens are estimated at 4 characters each.
651 */
652 const t3ContextReport = (model: Model) => {
653 const size = (chars: number) => `${chars} chars (~${Math.round(chars / 4)} tokens)`;
654 const descriptions = t3Tools.reduce((sum, tool) => sum + valueText(tool.description).length, 0);
655 const schemas = JSON.stringify(t3Tools).length;
656 return [
657 `- T3 context: system text ${size((isCodex(model) ? JSON.stringify(buildCodexAdditionalContext({ model }, { browser: true, device: true })) : t3SystemAppend(model)).length)}`,
658 `tool descriptions ${size(descriptions)}`,
659 `full tool schemas ${size(schemas)} across ${t3Tools.length} tools.`,
660 ].join(", ");
661 };
662
663 const stringProperty = (description: string) => ({ type: "string", description });
664 const NATIVE_SCHEMAS: Readonly<Record<string, Omit<ApiTool, "name">>> = {
665 WebFetch: {
666 description: "Fetches content from a URL and answers the prompt about it.",
667 input_schema: {
668 type: "object",
669 properties: {
670 url: stringProperty("The URL to fetch"),
671 prompt: stringProperty("What to extract"),
672 },
673 required: ["url", "prompt"],
674 },
675 },
676 WebSearch: {
677 description: "Searches the web and returns result titles, links, and snippets.",
678 input_schema: {
679 type: "object",
680 properties: { query: stringProperty("The search query") },
681 required: ["query"],
682 },
683 },
684 };
685
686 /** Rebuilds the parts of a captured request that every case shares. */
687 export const buildBase = (
688 template: typeof MessagesRequest.Type,
689 options: {
690 readonly model: Model;
691 readonly keepUserInstructions: boolean;
692 readonly subagent?: boolean;
693 readonly links?: boolean;
694 readonly forge?: boolean;
695 /** A native subagent starts from its task: T3 appends nothing to its system prompt. */
696 readonly nativeSubagent?: boolean;
697 /** false: the thread has no agent device access, so T3's text leaves devices out. */
698 readonly deviceAccess?: boolean;
699 /** false: the thread has no agent browser access, so T3's text leaves the browser out. */
700 readonly browserAccess?: boolean;
701 readonly arm?: Arm;
702 },
703 ) => {
704 const t3 = visibleT3Tools(options);
705 const firstAssistant = template.messages.findIndex((message) => message.role === "assistant");
706 const head = template.messages.slice(0, firstAssistant === -1 ? undefined : firstAssistant);
707 const systemTexts = [
708 template.system ?? [],
709 ...head.flatMap((m) => (m.role === "system" ? [m.content] : [])),
710 ]
711 .flatMap(texts)
712 .filter((value) => !value.startsWith("x-anthropic-billing-header:"));
713 const deferredList = DEFERRED_LIST.exec(systemTexts.join("\n"))?.[2];
714 if (systemTexts.filter((value) => value.includes(RUNTIME_MARKER)).length !== 1 || !deferredList) {
715 throw new Error(`The template needs one ${RUNTIME_MARKER} block and a deferred tool list.`);
716 }
717 const nativeDeferred = deferredList.split("\n").filter((name) => name && !isT3Name(name));
718 const names = [
719 ...nativeDeferred,
720 ...t3.filter((tool) => !alwaysLoaded.has(tool.name)).map((tool) => tool.name),
721 ]
722 .toSorted()
723 .join("\n");
724 const { model } = options;
725 const append = options.nativeSubagent
726 ? ""
727 : forArm(t3SystemAppend(model, options.deviceAccess, options.browserAccess), options.arm);
728 const modelLine = `You are powered by the model named ${MODELS[model]}. The exact model ID is ${model}.`;
729 const joined = systemTexts.join("\n");
730 // The capture's home directory names its user. Cases use `/home/user` paths instead.
731 const cwd = /Primary working directory: (\S+)/.exec(joined)?.[1] ?? "/home/user/app";
732 const home = /^\/(?:home|Users)\/[^/]+/.exec(cwd)?.[0];
733 const anonymize = (value: string) =>
734 home === undefined ? value : value.replaceAll(home, home.replace(/[^/]+$/, "user"));
735 const system = systemTexts.map((value) => {
736 // T3's append is the tail of its block: runtime info, PR linking, then orchestration.
737 const at = value.indexOf(RUNTIME_MARKER);
738 return anonymize(at === -1 ? value : value.slice(0, at) + append)
739 .replace(DEFERRED_LIST, (_, header: string) => `${header}${names}\n`)
740 .replace(MODEL_LINE, modelLine)
741 .replace(IDENTITY, NEUTRAL_IDENTITY);
742 });
743 const [first] = template.messages;
744 // The last block is the captured prompt.
745 const reminders = (first?.role === "user" ? texts(first.content).slice(0, -1) : []).filter(
746 (value) =>
747 options.keepUserInstructions || !value.includes("Codebase and user instructions are"),
748 );
749 // gitStatus names a real repo state. Each case gets one built from its fake world instead.
750 const gitStatusAt = reminders.findIndex((value) => value.includes("# gitStatus"));
751 // Claude Code sends no schema for tools it only names. Web tools get their usual one, the
752 // rest load open.
753 const open = { type: "object", additionalProperties: true };
754 const nativeSchema = (name: string) =>
755 NATIVE_SCHEMAS[name] ?? { description: name, input_schema: open };
756 return {
757 model,
758 system: system.map(text),
759 reminders: reminders.filter((_, i) => i !== gitStatusAt).map((value) => text(anonymize(value))),
760 gitStatusAt: gitStatusAt === -1 ? reminders.length : gitStatusAt,
761 /** The shell simulator's cwd and OS follow the template's environment section. */
762 environment: {
763 cwd: anonymize(cwd),
764 platform: /Platform: (\w+)/.exec(joined)?.[1] ?? "linux",
765 date: /Today's date is ([\d-]+)/.exec(reminders.join("\n"))?.[1],
766 },
767 nativeTools: [
768 ...template.tools.filter((tool) => !isT3Name(tool.name)),
769 ...t3.filter((tool) => alwaysLoaded.has(tool.name)),
770 ],
771 /** Tools ToolSearch loads: T3 tools from source, and native tools the template only names. */
772 t3,
773 deferred: [...t3, ...nativeDeferred.map((name) => ({ name, ...nativeSchema(name) }))],
774 };
775 };
776
777 type Base = ReturnType<typeof buildBase> & {
778 readonly codex?: typeof ResponsesRequest.Type;
779 /** What T3's SubagentStart hooks give a native subagent. */
780 readonly subagentContext?: string;
781 };
782
783 /** Keep the captured harness prompt and tools; replace app-owned context and the real user. */
784 const buildCodexBase = (
785 template: typeof ResponsesRequest.Type,
786 options: {
787 readonly model: Model;
788 readonly keepUserInstructions: boolean;
789 readonly effort?: string | undefined;
790 readonly subagent?: boolean;
791 readonly links?: boolean;
792 readonly forge?: boolean;
793 /** false: the thread has no agent device access, so T3's text leaves devices out. */
794 readonly deviceAccess?: boolean;
795 /** false: the thread has no agent browser access, so T3's text leaves the browser out. */
796 readonly browserAccess?: boolean;
797 readonly arm?: Arm;
798 },
799 ): Base => {
800 const t3 = visibleT3Tools(options);
801 const contentText = (item: Block) =>
802 Array.isArray(item.content)
803 ? item.content.map((block: Block) => valueText(block.text)).join("\n")
804 : valueText(item.content);
805 const joined = template.input.map(contentText).join("\n");
806 if (!joined.includes("<t3_code_runtime>"))
807 throw new Error("Codex template needs full T3 context.");
808 const cwd = /<cwd>(.*?)<\/cwd>/.exec(joined)?.[1] ?? "/home/user/app";
809 const home = /^\/(?:home|Users)\/[^/]+/.exec(cwd)?.[0];
810 const clean = (s: string) => (home ? s.replaceAll(home, home.replace(/[^/]+$/, "user")) : s);
811 const effort = options.effort ?? template.reasoning.effort;
812 const message = (value: string, role = "developer") => ({
813 type: "message",
814 role,
815 content: [{ type: "input_text", text: value }],
816 });
817 // The captured <collaboration_mode> message stays: Codex supplies its own mode text natively.
818 const input = template.input.flatMap((item) => {
819 const content = contentText(item);
820 if (content.includes("<t3_code_")) return [];
821 if (item.role === "user") {
822 const env = /<environment_context>[\s\S]*?<\/environment_context>/.exec(content)?.[0];
823 return env ? [message(clean(options.keepUserInstructions ? content : env), "user")] : [];
824 }
825 // Only harness messages and tools. A captured turn's answers and tool calls would leak into
826 // every case.
827 if (item.type !== "message" && item.type !== "additional_tools") return [];
828 if (item.role === "assistant") return [];
829 return [JSON.parse(clean(JSON.stringify(item))) as Block];
830 });
831 const context = buildCodexAdditionalContext(
832 { model: options.model, reasoningEffort: effort },
833 { browser: options.browserAccess ?? true, device: options.deviceAccess ?? true },
834 );
835 input.push(
836 ...Object.entries(context).map(([key, entry]) =>
837 message(`<${key}>${forArm(entry.value, options.arm)}</${key}>`),
838 ),
839 );
840 const nativeTools = input
841 .flatMap((item) =>
842 item.type === "additional_tools" && Array.isArray(item.tools)
843 ? item.tools.flatMap((ns: unknown) =>
844 isRecord(ns) && Array.isArray(ns.tools) ? ns.tools : [],
845 )
846 : [],
847 )
848 .filter((tool): tool is ApiTool => isRecord(tool) && typeof tool.name === "string");
849 if (!nativeTools.some((tool) => tool.name === "exec"))
850 throw new Error("Codex template needs code-mode exec.");
851 return {
852 model: options.model,
853 system: [],
854 reminders: [],
855 gitStatusAt: 0,
856 environment: {
857 cwd: clean(cwd),
858 platform: cwd.startsWith("/Users/") ? "darwin" : "linux",
859 date: /<current_date>(.*?)<\/current_date>/.exec(joined)?.[1],
860 },
861 nativeTools,
862 t3,
863 deferred: [...t3],
864 codex: { ...template, input, reasoning: { ...template.reasoning, effort } },
865 };
866 };
867
868 /** Replay prior calls in Codex's wire format, including Claude-shaped file operations. */
869 const codexCall = (name: string, input: unknown) => {
870 const args = isRecord(input) ? input : {};
871 if (name === "Bash")
872 return `text(await tools.exec_command(${JSON.stringify({ ...args, cmd: args.command })}));`;
873 if (["Read", "Write", "Edit"].includes(name)) {
874 const file = valueText(args.file_path);
875 const command =
876 name === "Read"
877 ? `cat ${JSON.stringify(file)}`
878 : name === "Write"
879 ? `cat > ${JSON.stringify(file)} <<'EVAL_EOF'\n${valueText(args.content)}\nEVAL_EOF`
880 : `python3 -c ${JSON.stringify(`from pathlib import Path; p=Path(${JSON.stringify(file)}); p.write_text(p.read_text().replace(${JSON.stringify(args.old_string)}, ${JSON.stringify(args.new_string)}))`)}`;
881 return `text(await tools.exec_command(${JSON.stringify({ cmd: command })}));`;
882 }
883 return `text(await tools.${name.replace("mcp__t3-code__", "mcp__t3_code__")}(${JSON.stringify(input)}));`;
884 };
885 const codexMessages = (messages: ReadonlyArray<ApiMessage>): Array<Block> =>
886 messages.flatMap((message) =>
887 message.content.map((block): Block =>
888 block.type === "tool_use"
889 ? {
890 type: "custom_tool_call",
891 call_id: block.id,
892 name: "exec",
893 input: codexCall(valueText(block.name), block.input),
894 }
895 : block.type === "tool_result"
896 ? {
897 type: "custom_tool_call_output",
898 call_id: block.tool_use_id,
899 output: valueText(block.content),
900 }
901 : {
902 type: "message",
903 role: message.role,
904 content: [
905 {
906 type: message.role === "user" ? "input_text" : "output_text",
907 text: valueText(block.text),
908 },
909 ],
910 },
911 ),
912 );
913 /** One tool call the shell simulator has answered, or one from the case's prior turns. */
914 type ShellEntry = { readonly tool: string; readonly input: unknown; readonly output: string };
915 /** PRs T3 links by itself because a shell command created them, as `gh pr create` does. */
916 const createdPullRequests = ({ tool, input, output }: ShellEntry) =>
917 tool === "Bash" && isRecord(input) && typeof input.command === "string"
918 ? changeRequestsCreatedBy(input.command, output).map(({ url }) => url)
919 : [];
920
921 /** The PR number a shell command opened for `branch`; `--head` naming another branch is not it. */
922 const branchPullRequestCreatedBy = (entry: ShellEntry, branch: string | undefined) => {
923 const command = isRecord(entry.input) ? valueText(entry.input.command) : "";
924 const head = /--head[=\s]+["']?([^\s"']+)/.exec(command)?.[1];
925 if (head !== undefined && branch !== undefined && head.split(":").at(-1) !== branch) {
926 return undefined;
927 }
928 return createdPullRequests(entry)
929 .map((url) => Number(/\/pull\/(\d+)/.exec(url)?.[1]))
930 .find((number) => number > 0);
931 };
932 /** The branch the agent last checked out or switched to in the shell, else the world's. */
933 const checkedOutBranch = (sample: Sample) => {
934 let branch = sample.testCase.setup.world?.branch ?? "";
935 for (const entry of sample.shell) {
936 for (const part of shellParts(entry)) branch = switchedTo(part) ?? branch;
937 }
938 return branch;
939 };
940 /** A shell entry's commands, split at separators, with fd redirects dropped. */
941 const shellParts = (entry: ShellEntry) =>
942 (isRecord(entry.input) ? valueText(entry.input.command) : "")
943 .replace(/\s\d?>&\d/g, "")
944 .split(/[;&|\n]+/);
945 /** The branch one command checks out or switches to, if it does. */
946 const switchedTo = (part: string) => {
947 // `git checkout <ref> -- <paths>` restores files and stays on the branch.
948 if (/\s--(\s|$)/.test(part)) return undefined;
949 return /\bgit (?:checkout|switch)\s+(?:-[bcBC]\s+)?([A-Za-z0-9][\w./-]*)\s*$/.exec(
950 part.trim(),
951 )?.[1];
952 };
953
954 /**
955 * One sample's state. ToolSearch adds loaded schemas to `tools`, like Claude Code. `shell` is the
956 * history the simulator keeps consistent with; `linked`, `nextTask` and `threads` do the same for
957 * T3 results.
958 */
959 type Sample = {
960 readonly testCase: McpToolUseCase;
961 readonly base: Base;
962 readonly tools: Array<ApiTool>;
963 readonly repo: string;
964 readonly shell: Array<ShellEntry>;
965 readonly shellUsage: Usage;
966 readonly linked: Set<string>;
967 /** The branch PR the agent opened in this sample, which links alone do not prove. */
968 openedPullRequest: number | undefined;
969 /** PRs pull_request_create opened from branches other than the world's, by branch. */
970 readonly openedByBranch: Map<string, number>;
971 /** PR changes the agent made through T3's tools, which gh and the host show from then on. */
972 readonly forgeChanges: Array<string>;
973 /** PRs pull_request_merge merged, as `repo#number`, which reads show as merged from then on. */
974 readonly merged: Set<string>;
975 readonly codeStore: Record<string, unknown>;
976 nextTask: number;
977 /** Threads the agent launched in this sample, by id, so reading one shows its own brief. */
978 readonly threads: Map<string, { readonly title: string; readonly brief: string }>;
979 /** Files T3's request_secret saved in this sample, in call order. */
980 readonly secretFiles: Array<string>;
981 /** Files request_secret saved for `secret: false` fields, with their labels. */
982 readonly plainFiles: Array<{ readonly file: string; readonly label: string }>;
983 secretRequests: number;
984 /** T3's device tools, once `withDevices` builds them for a world with devices. */
985 devices?: DeviceTools;
986 /** T3's results for prior device calls whose case result is "default". */
987 readonly priorResults: Map<unknown, string>;
988 /** Prior calls whose T3 result is an error, so the agent sees it as one. */
989 readonly priorErrors: Set<unknown>;
990 /** T3's t3_background_wait, once `withBackgroundWait` builds it for `world.backgroundTasks`. */
991 backgroundWait?: BackgroundWait;
992 /** Calls so far per mocked tool, for mocks that answer in sequence. */
993 readonly mockCalls: Map<string, number>;
994 };
995
996 const SHELL_TOOLS = new Set(["Bash", "Read", "Edit", "Write", "WebFetch"]);
997
998 export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => {
999 const prior = testCase.setup.priorTurns ?? [];
1000 const priorCalls = prior.flatMap((turn) => turn.toolCalls ?? []);
1001 const repo = testCase.setup.world?.repo ?? "acme/app";
1002 const urls = [
1003 ...(testCase.setup.world?.linkedPullRequests ?? []).map(
1004 ({ number }) => pullRequest(repo, { number }).url,
1005 ),
1006 ...priorCalls.flatMap((call) =>
1007 ["link_pull_request", "watch_pull_request"].includes(call.tool) && isRecord(call.input)
1008 ? [pullRequest(repo, call.input).url]
1009 : call.tool === "link_pull_requests" &&
1010 isRecord(call.input) &&
1011 Array.isArray(call.input.urls)
1012 ? call.input.urls
1013 .filter((url) => PULL_REQUEST_URL.test(valueText(url)))
1014 .map((url) => pullRequest(repo, { url }).url)
1015 : [],
1016 ),
1017 ];
1018 const taskNumbers = [...valueText(prior).matchAll(/task-(\d+)/g)].map((match) =>
1019 Number(match[1]),
1020 );
1021 const shell = priorCalls
1022 .filter((call) => SHELL_TOOLS.has(call.tool))
1023 .map(({ tool, input, result }) => ({ tool, input, output: valueText(result) }));
1024 return {
1025 testCase,
1026 base,
1027 tools: [...base.nativeTools],
1028 repo: testCase.setup.world?.repo ?? "acme/app",
1029 shell,
1030 shellUsage: emptyUsage(),
1031 linked: new Set([...urls, ...shell.flatMap(createdPullRequests)]),
1032 openedByBranch: new Map(),
1033 forgeChanges: [],
1034 merged: new Set(),
1035 openedPullRequest:
1036 shell
1037 .map((entry) => branchPullRequestCreatedBy(entry, testCase.setup.world?.branch))
1038 .find((number) => number !== undefined) ??
1039 priorCalls
1040 .filter((call) => call.tool === "pull_request_create")
1041 .map((call) => Number(/\/pull\/(\d+)/.exec(valueText(call.result))?.[1]))
1042 .find((number) => number > 0),
1043 codeStore: {},
1044 nextTask: Math.max(0, ...taskNumbers) + 1,
1045 threads: new Map(),
1046 secretFiles: [],
1047 plainFiles: [],
1048 secretRequests: 0,
1049 priorResults: new Map(),
1050 priorErrors: new Set(),
1051 mockCalls: new Map(),
1052 };
1053 };
1054
1055 /** A `{"$sequence": [...]}` mock answers its nth call with the nth result, then repeats the last. */
1056 const nextMock = (sample: Sample, name: string, mock: unknown) => {
1057 if (!isRecord(mock) || !Array.isArray(mock.$sequence) || mock.$sequence.length === 0) return mock;
1058 const n = sample.mockCalls.get(name) ?? 0;
1059 sample.mockCalls.set(name, n + 1);
1060 return mock.$sequence[Math.min(n, mock.$sequence.length - 1)];
1061 };
1062
1063 /**
1064 * Builds T3's device tools for a world with devices. Prior device calls whose result is "default"
1065 * run first, in order and without the case's mocks, so their results and sessions are T3's. As in
1066 * the real flow, device_open starts no agent-device session: one starts at a launcher `open` in the
1067 * sample's shell history, prior turns included, or where `world.agentDeviceSessions` declares it.
1068 */
1069 export const withDevices = Effect.fn("evaluateMcpToolUse.withDevices")(function* (sample: Sample) {
1070 const world = sample.testCase.setup.world;
1071 if (world?.devices === undefined) return;
1072 sample.devices = yield* makeDeviceTools(
1073 { ...world, devices: world.devices },
1074 sample.base.environment,
1075 );
1076 for (const call of (sample.testCase.setup.priorTurns ?? []).flatMap((t) => t.toolCalls ?? [])) {
1077 if (call.result !== "default" || !isDeviceTool(call.tool)) continue;
1078 sample.priorResults.set(call, (yield* sample.devices.call(call.tool, call.input)).text);
1079 }
1080 });
1081
1082 /**
1083 * Builds T3's t3_background_wait for a world with background tasks, and answers prior waits whose
1084 * result is "default" with T3's result.
1085 */
1086 export const withBackgroundWait = Effect.fn("evaluateMcpToolUse.withBackgroundWait")(function* (
1087 sample: Sample,
1088 ) {
1089 const running = sample.testCase.setup.world?.backgroundTasks;
1090 if (running === undefined) return;
1091 const backgroundWait = yield* makeBackgroundWait(running);
1092 sample.backgroundWait = backgroundWait;
1093 for (const call of (sample.testCase.setup.priorTurns ?? []).flatMap((t) => t.toolCalls ?? [])) {
1094 if (call.result !== "default" || call.tool !== "t3_background_wait") continue;
1095 const result = yield* backgroundWait(call.input);
1096 sample.priorResults.set(call, result.text);
1097 if (result.isError) sample.priorErrors.add(call);
1098 }
1099 });
1100
1101 // Main's history under every case's world.
1102 const MAIN_COMMITS = [
1103 { sha: "9f2c1ab", subject: "feat(web): show the model name in the thread header" },
1104 { sha: "4e7d2c0", subject: "fix(server): retry provider start once on ENOENT" },
1105 { sha: "b81f3d9", subject: "chore: update dependencies" },
1106 ];
1107
1108 /**
1109 * The gitStatus Claude Code shows when a session starts, built from the case's world. Commits and
1110 * a branch that the prior turns made (their tool results show it) came later, so they are left out.
1111 */
1112 const gitStatus = ({ setup }: McpToolUseCase) => {
1113 const prior = valueText(setup.priorTurns ?? []);
1114 const branch = setup.world?.branch ?? "main";
1115 const commits = [...(setup.world?.commits ?? []), ...MAIN_COMMITS].filter(
1116 (commit) => !prior.includes(` ${commit.sha}] `),
1117 );
1118 return [
1119 "As you answer the user's questions, you can use the following context:",
1120 "# gitStatus",
1121 "This is the git status at the start of the conversation. Note that this status is a snapshot in time, and will not update during the conversation.",
1122 "",
1123 `Current branch: ${prior.includes(`Switched to a new branch '${branch}'`) ? "main" : branch}`,
1124 "",
1125 "Main branch (you will usually use this for PRs): main",
1126 "",
1127 "Status:",
1128 setup.world?.status || "(clean)",
1129 "",
1130 "Recent commits:",
1131 ...commits.slice(0, 5).map((commit) => `${commit.sha} ${commit.subject}`),
1132 "",
1133 "Claude Code attached this context automatically; it isn't part of the user's message. It describes the user's own account and workspace, so they don't need it reported back.",
1134 ].join("\n");
1135 };
1136
1137 const findTool = (base: Base, bare: string) =>
1138 [...base.nativeTools, ...base.deferred].find((tool) => normalizeToolName(tool.name) === bare) ??
1139 (base.codex ? { name: bare } : undefined);
1140
1141 /** Adds deferred schemas to `tools` and returns the ToolSearch result Claude Code sends. */
1142 const loadTools = (tools: Array<ApiTool>, id: string, found: ReadonlyArray<ApiTool>): Block => {
1143 for (const tool of found) {
1144 if (!hasTool(tools, tool.name)) tools.push({ ...tool, defer_loading: true });
1145 }
1146 const content = found.map((tool) => ({ type: "tool_reference", tool_name: tool.name }));
1147 return { type: "tool_result", tool_use_id: id, content };
1148 };
1149
1150 /** Messages for a case: reminders, context, prior turns with real tool blocks, then the prompt. */
1151 export const buildMessages = (sample: Sample) => {
1152 const { testCase, base, tools } = sample;
1153 const messages: Array<ApiMessage> = [];
1154 const push = (role: ApiMessage["role"], ...blocks: ReadonlyArray<Block>) => {
1155 const last = messages.at(-1);
1156 if (last?.role !== role) messages.push({ role, content: blocks });
1157 else messages[messages.length - 1] = { role, content: [...last.content, ...blocks] };
1158 };
1159 const { role, context, delegateRole, priorTurns = [] } = testCase.setup;
1160 const sideChat = testCase.setup.sideChat;
1161 if (testCase.prompt === "" && (role !== "top" || sideChat !== undefined)) {
1162 throw new Error(
1163 `${testCase.id}: Empty-prompt replay requires a top-level thread without side-chat context.`,
1164 );
1165 }
1166 const sideChatContext =
1167 sideChat === undefined
1168 ? undefined
1169 : makeSideChatContext({
1170 parentThreadId: ThreadId.make(sideChat.parentThreadId),
1171 digest: sideChat.digest,
1172 });
1173 const native = role === "native-subagent";
1174 const notes = [
1175 native
1176 ? "You are a subagent started by the parent agent's Agent tool. Your final message goes to the parent agent, not to the user."
1177 : undefined,
1178 context,
1179 ];
1180 const reminders = base.reminders.toSpliced(base.gitStatusAt, 0, reminder(gitStatus(testCase)));
1181 push("user", ...reminders, ...notes.flatMap((note) => (note ? [reminder(note)] : [])));
1182 const toolSearch = findTool(base, "ToolSearch")?.name ?? "ToolSearch";
1183 let nextId = 0;
1184 for (const turn of priorTurns) {
1185 const calls = (turn.toolCalls ?? []).map((call) => {
1186 const tool = findTool(base, call.tool);
1187 if (tool === undefined) throw new Error(`${testCase.id}: unknown prior tool ${call.tool}.`);
1188 const builder = isDeviceTool(call.tool)
1189 ? "withDevices"
1190 : call.tool === "t3_background_wait"
1191 ? "withBackgroundWait"
1192 : undefined;
1193 if (call.result === "default" && builder && !sample.priorResults.has(call)) {
1194 throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run ${builder}.`);
1195 }
1196 return { ...call, id: `toolu_prior_${nextId++}`, name: tool.name, tool, prior: call };
1197 });
1198 // Prior T3 calls load their schemas first, as in a real deferred session.
1199 const unloaded = [...new Set(calls.map((call) => call.tool))].filter(
1200 (tool) => !hasTool(tools, tool.name),
1201 );
1202 if (unloaded.length > 0 && !base.codex) {
1203 const id = `toolu_prior_${nextId++}`;
1204 const query = `select:${unloaded.map((tool) => tool.name).join(",")}`;
1205 const input = { query, max_results: unloaded.length };
1206 push("assistant", { type: "tool_use", id, name: toolSearch, input });
1207 push("user", loadTools(tools, id, unloaded), text(TOOL_LOADED));
1208 }
1209 // One call per step, as most real turns run them.
1210 for (const { id, name, input, result, tool, prior } of calls) {
1211 push("assistant", { type: "tool_use", id, name, input });
1212 const replayed =
1213 normalizeToolName(tool.name) === "orchestrator_capabilities"
1214 ? capabilities(sample, result)
1215 : (sample.priorResults.get(prior) ?? result);
1216 const answer = toolResult(id, replayed);
1217 push("user", sample.priorErrors.has(prior) ? { ...answer, is_error: true } : answer);
1218 }
1219 if (turn.text) push(turn.role, text(turn.text));
1220 }
1221 const userText =
1222 testCase.promptContext === undefined
1223 ? testCase.prompt
1224 : projectComposerContextForProvider({ text: testCase.prompt, ...testCase.promptContext });
1225 const prompt =
1226 sideChatContext === undefined ? userText : withInlineContext(sideChatContext, userText);
1227 const childPrompt = () => taskPrompt({ task: prompt, role: delegateRole });
1228 if (testCase.prompt !== "")
1229 push("user", text(role === "delegated-child" ? childPrompt() : prompt));
1230 // Claude Code gives a subagent its SubagentStart hooks' context as a reminder after the task.
1231 if (native && base.subagentContext) {
1232 push("user", reminder(`SubagentStart hook additional context: ${base.subagentContext}`));
1233 }
1234 return messages;
1235 };
1236
1237 // Results for T3 tools a case does not mock. The model ids are public in this repo's catalog.
1238 const claudeModel = (id: string) => ({ id, options: [{ id: "effort", values: EFFORTS }] });
1239 const codexModel = (id: string) => ({ id, options: [{ id: "reasoningEffort", values: EFFORTS }] });
1240 // Mirrors the real catalog's main models, so a case's named model is not missing by accident.
1241 const PROVIDERS = [
1242 {
1243 providerInstanceId: "claudeAgent",
1244 models: ["claude-opus-5-5", "claude-sonnet-5-5", "claude-fable-5-2", "claude-haiku-5-5"].map(
1245 claudeModel,
1246 ),
1247 },
1248 {
1249 providerInstanceId: "codex",
1250 models: ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-luna"].map(codexModel),
1251 },
1252 { providerInstanceId: "muse", models: ["muse-wonder-full", "muse-wonder"].map(codexModel) },
1253 { providerInstanceId: "grok", models: ["grok-4.7"].map(codexModel) },
1254 ];
1255 const uuid = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
1256 const DEFAULT_THREAD = {
1257 title: "Fix the flaky reconnect test",
1258 status: "idle",
1259 items: [
1260 {
1261 position: 1,
1262 createdBy: "user",
1263 text: "The reconnect test in src/session.test.ts fails about 1 in 10 runs. Fix it.",
1264 },
1265 {
1266 position: 2,
1267 createdBy: "assistant",
1268 text: "Fixed. The test now waits for the socket close event before it reconnects. 20 runs in a row pass.",
1269 },
1270 ],
1271 };
1272 const HTML_RENDERED =
1273 "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.";
1274
1275 const PULL_REQUEST_URL = /\/pull\/\d+\/?$/;
1276
1277 /** Pull request identity from a `url` or `repository` plus `number` input, like T3's results. */
1278 const pullRequest = (repo: string, input: Readonly<Record<string, unknown>>) => {
1279 const url = typeof input.url === "string" ? input.url : "";
1280 const repository =
1281 /github\.com\/([^/]+\/[^/]+)\/pull/.exec(url)?.[1] ?? valueText(input.repository ?? repo);
1282 const number = Number(/(\d+)\/?$/.exec(url)?.[1] ?? input.number);
1283 return {
1284 host: "github.com",
1285 repository,
1286 number,
1287 url: url || `https://github.com/${repository}/pull/${number}`,
1288 };
1289 };
1290
1291 /** A linked PR as `list_thread_pull_requests` shows it. Links made in the sample are open. */
1292 const linkedEntry = (sample: Sample, url: string) => {
1293 const pr = pullRequest(sample.repo, { url });
1294 const seeded = sample.testCase.setup.world?.linkedPullRequests?.find(
1295 ({ number }) => number === pr.number && pr.repository === sample.repo,
1296 );
1297 const state = mergedIn(sample, pr.repository, pr.number) ? "merged" : (seeded?.state ?? "open");
1298 return { ...pr, state, title: seeded?.title ?? null };
1299 };
1300
1301 const mergedIn = (sample: Sample, repository: string, number: unknown) =>
1302 sample.merged.has(`${repository.toLowerCase()}#${Number(number)}`);
1303
1304 /** The world's PRs. `prs` is free-form world data: number, title, headRefName, checks. */
1305 const worldPullRequests = (sample: Sample) => {
1306 const prs = sample.testCase.setup.world?.prs;
1307 return Array.isArray(prs) ? prs.filter(isRecord) : [];
1308 };
1309
1310 /** A world PR's `reviewThreads`: id, path, line, isResolved, comments of author and body. */
1311 const worldReviewThreads = (pr: Readonly<Record<string, unknown>> | undefined) =>
1312 Array.isArray(pr?.reviewThreads) ? pr.reviewThreads.filter(isRecord) : [];
1313
1314 /** World fixtures spell states as GitHub does (`MERGED`) or as T3 does (`merged`). */
1315 const worldState = (pr: Readonly<Record<string, unknown>> | undefined) => {
1316 const state = valueText(pr?.state ?? "open").toLowerCase();
1317 return state === "merged" || state === "closed" ? state : "open";
1318 };
1319
1320 /**
1321 * The PR for the sample's branch: the world's, else the one a create would open. A case that
1322 * mocks `gh pr create` names that URL, so the T3 tool opens the same number.
1323 */
1324 const worldBranchPullRequest = (sample: Sample) => {
1325 const branch = sample.testCase.setup.world?.branch;
1326 const prs = worldPullRequests(sample);
1327 const own = prs.find((pr) => pr.headRefName === branch);
1328 if (own !== undefined) return { number: Number(own.number), created: false, pr: own };
1329 if (sample.openedPullRequest !== undefined) {
1330 return { number: sample.openedPullRequest, created: false, pr: undefined };
1331 }
1332 const mocked = (sample.testCase.setup.shellMocks ?? []).find((mock) =>
1333 /\b(pr|mr) create\b/.test(mock.match),
1334 );
1335 const fromMock = Number(/\/pull\/(\d+)/.exec(mocked?.output ?? "")?.[1]);
1336 const number =
1337 fromMock > 0 ? fromMock : Math.max(123, ...prs.map((pr) => Number(pr.number) || 0)) + 1;
1338 return { number, created: true, pr: undefined };
1339 };
1340
1341 /** The PR for the checked-out branch: a stack layer's own, or the world branch's. */
1342 const branchPullRequest = (sample: Sample) => {
1343 const checkedOut = checkedOutBranch(sample);
1344 if (checkedOut === sample.testCase.setup.world?.branch) return worldBranchPullRequest(sample);
1345 const known = worldPullRequests(sample).find((pr) => pr.headRefName === checkedOut);
1346 if (known !== undefined) return { number: Number(known.number), created: false, pr: known };
1347 const opened = sample.openedByBranch.get(checkedOut);
1348 if (opened !== undefined) return { number: opened, created: false, pr: undefined };
1349 const number =
1350 Math.max(
1351 worldBranchPullRequest(sample).number,
1352 ...sample.openedByBranch.values(),
1353 ...worldPullRequests(sample).map((pr) => Number(pr.number) || 0),
1354 ...[...sample.linked].map((url) => Number(/\/pull\/(\d+)/.exec(url)?.[1]) || 0),
1355 ) + 1;
1356 return { number, created: true, pr: undefined };
1357 };
1358
1359 /** PullRequestService reads this many logs when no check is named. */
1360 const DEFAULT_CHECK_LOGS = 5;
1361
1362 /** The checks `ci_logs` and `ci_rerun` act on, selected as PullRequestService does. */
1363 const selectedWorldChecks = (sample: Sample, input: Readonly<Record<string, unknown>>) => {
1364 const number = typeof input.number === "number" ? input.number : branchPullRequest(sample).number;
1365 const world = worldPullRequests(sample).find((pr) => Number(pr.number) === number);
1366 const all = Array.isArray(world?.checks) ? world.checks.filter(isRecord) : [];
1367 const named = typeof input.check === "string" ? input.check.trim().toLowerCase() : null;
1368 const checks = all.filter((check) =>
1369 named === null
1370 ? check.state === "fail" || check.state === "cancelled"
1371 : valueText(check.name).toLowerCase() === named,
1372 );
1373 return { number, all, checks, named };
1374 };
1375
1376 /** A believable T3 result for a tool the case does not mock. */
1377 export const t3Default = (
1378 sample: Sample,
1379 name: string,
1380 input: Readonly<Record<string, unknown>>,
1381 ) => {
1382 const pr = () => pullRequest(sample.repo, input);
1383 const newThread = (request: unknown, brief: unknown) => {
1384 const n = sample.nextTask++;
1385 const title =
1386 isRecord(request) && typeof request.title === "string" ? request.title : "New thread";
1387 const text = typeof brief === "string" ? brief.trim() : "";
1388 sample.threads.set(uuid(n), { title, brief: text });
1389 return {
1390 threadId: uuid(n),
1391 title,
1392 ...(text ? { runId: `run-${n}`, status: "starting" } : { status: "idle" }),
1393 };
1394 };
1395 switch (name) {
1396 case "t3_thread_launch":
1397 return newThread(input, input.message);
1398 case "create_threads":
1399 return {
1400 threads: (Array.isArray(input.threads) ? input.threads : []).map((request) => {
1401 const target = isRecord(request) && isRecord(request.target) ? request.target : {};
1402 return {
1403 ...newThread(request, isRecord(request) ? request.prompt : undefined),
1404 createdBy: "agent",
1405 creationSource: "mcp",
1406 providerInstanceId:
1407 target.providerInstanceId ?? (sample.base.codex ? "codex" : "claudeAgent"),
1408 model: target.model ?? sample.base.model,
1409 };
1410 }),
1411 };
1412 case "orchestrator_capabilities":
1413 return {
1414 inheritedProviderInstanceId: "claudeAgent",
1415 inheritedModel: sample.base.model,
1416 providers: PROVIDERS,
1417 };
1418 case "delegate_task": {
1419 const n = sample.nextTask++;
1420 const target = isRecord(input.target) ? input.target : {};
1421 return {
1422 taskId: `task-${n}`,
1423 childThreadId: uuid(n),
1424 childRunId: `run-${n}`,
1425 status: "running",
1426 workState: "working",
1427 hasPendingChildRuns: false,
1428 providerInstanceId:
1429 target.providerInstanceId ?? (sample.base.codex ? "codex" : "claudeAgent"),
1430 model: target.model ?? sample.base.model,
1431 summary: null,
1432 hasPendingQuestion: false,
1433 pendingApproval: null,
1434 };
1435 }
1436 case "task_status":
1437 return {
1438 taskId: input.taskId,
1439 status: "running",
1440 workState: "working",
1441 hasPendingChildRuns: false,
1442 summary: null,
1443 };
1444 case "task_cancel":
1445 return { taskId: input.taskId, status: "cancel_requested" };
1446 case "link_pull_request": {
1447 const alreadyLinked = sample.linked.has(pr().url);
1448 sample.linked.add(pr().url);
1449 return { ...pr(), alreadyLinked };
1450 }
1451 case "link_pull_requests": {
1452 const urls = Array.isArray(input.urls) ? input.urls.map(valueText) : [];
1453 const invalidUrls = urls.filter((url) => !PULL_REQUEST_URL.test(url));
1454 const targets = [...new Set(urls.filter((url) => PULL_REQUEST_URL.test(url)))].map((url) =>
1455 pullRequest(sample.repo, { url }),
1456 );
1457 const fresh = targets.filter(({ url }) => !sample.linked.has(url));
1458 for (const { url } of targets) sample.linked.add(url);
1459 const byRepository = (prs: ReadonlyArray<ReturnType<typeof pullRequest>>) =>
1460 [...Map.groupBy(prs, (pr) => pr.repository).values()].map((group) => ({
1461 host: "github.com",
1462 repository: group[0]!.repository,
1463 numbers: group.map((pr) => pr.number),
1464 }));
1465 return {
1466 linked: byRepository(fresh),
1467 alreadyLinked: byRepository(targets.filter((pr) => !fresh.includes(pr))),
1468 linkedCount: targets.length,
1469 ...(invalidUrls.length > 0 ? { invalidUrls } : {}),
1470 };
1471 }
1472 case "pull_request_create": {
1473 const checkedOut = checkedOutBranch(sample);
1474 const { number, created } = branchPullRequest(sample);
1475 const target = pullRequest(sample.repo, { number });
1476 sample.linked.add(target.url);
1477 // Another layer of a stack gets its own PR, opened once per branch like the real tool.
1478 if (checkedOut !== sample.testCase.setup.world?.branch) {
1479 sample.openedByBranch.set(checkedOut, number);
1480 } else if (created) sample.openedPullRequest = number;
1481 return { ...target, created };
1482 }
1483 case "pull_request_read": {
1484 const own = branchPullRequest(sample);
1485 const number = typeof input.number === "number" ? input.number : own.number;
1486 const world = worldPullRequests(sample).find((pr) => Number(pr.number) === number);
1487 const include = Array.isArray(input.include) ? input.include : [];
1488 const checks = Array.isArray(world?.checks) ? world.checks.filter(isRecord) : [];
1489 const checkRuns = checks.map((check) => ({
1490 name: valueText(check.name),
1491 status:
1492 check.state === "fail"
1493 ? "failure"
1494 : check.state === "cancelled" ||
1495 check.state === "pending" ||
1496 check.state === "action-required"
1497 ? check.state
1498 : "success",
1499 description: null,
1500 url: null,
1501 }));
1502 const comments = (Array.isArray(world?.comments) ? world.comments : []).filter(isRecord);
1503 return {
1504 ...pullRequest(sample.repo, { number }),
1505 title: valueText(world?.title ?? sample.testCase.setup.world?.commits?.[0]?.subject),
1506 state: mergedIn(sample, sample.repo, number) ? "merged" : worldState(world),
1507 isDraft: false,
1508 author: "user",
1509 headBranch: valueText(world?.headRefName ?? sample.testCase.setup.world?.branch),
1510 baseBranch: valueText(world?.baseRefName ?? "main"),
1511 // Only the branch's own PR is at the checkout's latest commit.
1512 headSha:
1513 typeof world?.headRefOid === "string"
1514 ? world.headRefOid
1515 : number === own.number
1516 ? (sample.testCase.setup.world?.commits?.[0]?.sha ?? null)
1517 : null,
1518 mergeability: valueText(world?.mergeability ?? "mergeable"),
1519 behindBy: typeof world?.behindBy === "number" ? world.behindBy : 0,
1520 autoMergeEnabled: false,
1521 checks: {
1522 passed: checkRuns.filter((check) => check.status === "success").length,
1523 pending: checkRuns.filter(
1524 (check) => check.status === "pending" || check.status === "action-required",
1525 ).length,
1526 failed: checkRuns
1527 .filter((check) => check.status === "failure" || check.status === "cancelled")
1528 .map((check) => check.name),
1529 },
1530 unresolvedThreads: worldReviewThreads(world).filter((thread) => thread.isResolved !== true)
1531 .length,
1532 ...(include.includes("body") ? { body: valueText(world?.body ?? "") } : {}),
1533 ...(include.includes("checks") ? { checkRuns } : {}),
1534 ...(include.includes("diff")
1535 ? { diff: { patch: valueText(world?.diff ?? ""), truncated: false, nextCursor: null } }
1536 : {}),
1537 ...(include.includes("threads") || include.includes("comments")
1538 ? {
1539 conversation: {
1540 ...(include.includes("threads")
1541 ? {
1542 reviewThreads: worldReviewThreads(world)
1543 .filter(
1544 (thread) => input.includeResolved === true || thread.isResolved !== true,
1545 )
1546 .map((thread) => ({
1547 id: valueText(thread.id),
1548 path: valueText(thread.path),
1549 line: typeof thread.line === "number" ? thread.line : null,
1550 isResolved: thread.isResolved === true,
1551 isOutdated: false,
1552 comments: (Array.isArray(thread.comments) ? thread.comments : [])
1553 .filter(isRecord)
1554 .map((comment) => ({
1555 author: valueText(comment.author),
1556 body: valueText(comment.body),
1557 createdAt: "2026-10-27T10:00:00Z",
1558 })),
1559 moreComments: false,
1560 })),
1561 }
1562 : {}),
1563 ...(include.includes("comments")
1564 ? {
1565 comments: comments.map((comment, index) => ({
1566 id: `comment-${index + 1}`,
1567 kind: "issue-comment",
1568 author: valueText(comment.author),
1569 body: valueText(comment.body),
1570 createdAt: "2026-10-27T10:00:00Z",
1571 path: null,
1572 reviewState: null,
1573 })),
1574 }
1575 : {}),
1576 truncated: false,
1577 nextCursor: null,
1578 },
1579 }
1580 : {}),
1581 };
1582 }
1583 case "pull_request_list": {
1584 const state = valueText(input.state ?? "open");
1585 const stateOf = (pr: Readonly<Record<string, unknown>>) =>
1586 mergedIn(sample, sample.repo, pr.number) ? "merged" : worldState(pr);
1587 return {
1588 pullRequests: worldPullRequests(sample)
1589 .filter((pr) => state === "all" || stateOf(pr) === state)
1590 .map((pr) => ({
1591 number: Number(pr.number),
1592 title: valueText(pr.title),
1593 url: pullRequest(sample.repo, { number: pr.number }).url,
1594 author: "user",
1595 headBranch: valueText(pr.headRefName),
1596 baseBranch: "main",
1597 state: stateOf(pr),
1598 isDraft: false,
1599 reviewDecision: null,
1600 checksState: null,
1601 updatedAt: "2026-10-27T10:00:00Z",
1602 })),
1603 truncated: false,
1604 nextCursor: null,
1605 };
1606 }
1607 case "pull_request_update":
1608 case "pull_request_review":
1609 return pullRequest(sample.repo, { number: input.number ?? branchPullRequest(sample).number });
1610 case "pull_request_respond":
1611 return {
1612 ...pullRequest(sample.repo, { number: input.number ?? branchPullRequest(sample).number }),
1613 replied: input.body !== undefined,
1614 resolved: typeof input.resolved === "boolean" ? input.resolved : null,
1615 };
1616 case "pull_request_merge": {
1617 const number = input.number ?? branchPullRequest(sample).number;
1618 if (input.auto !== true) {
1619 const layers = worldPullRequests(sample).map((pr) => Number(pr.number));
1620 const below =
1621 input.stack === true && sample.testCase.setup.world?.stack !== undefined
1622 ? layers.slice(0, layers.indexOf(Number(number)) + 1)
1623 : [];
1624 for (const merged of [Number(number), ...below])
1625 sample.merged.add(`${sample.repo.toLowerCase()}#${merged}`);
1626 }
1627 return {
1628 ...pullRequest(sample.repo, { number }),
1629 state: input.auto === true ? "open" : "merged",
1630 };
1631 }
1632 case "ci_logs":
1633 case "ci_rerun": {
1634 const { number, checks, named } = selectedWorldChecks(sample, input);
1635 return {
1636 ...pullRequest(sample.repo, { number }),
1637 checks: checks.map((check, index) =>
1638 name === "ci_rerun"
1639 ? { name: valueText(check.name), rerun: true, detail: null }
1640 : named === null && index >= DEFAULT_CHECK_LOGS
1641 ? {
1642 name: valueText(check.name),
1643 url: null,
1644 log: null,
1645 truncated: false,
1646 detail: "Not read, to keep the reply short. Ask for this check by name.",
1647 }
1648 : {
1649 name: valueText(check.name),
1650 url: null,
1651 log: valueText(check.log ?? `Error: ${valueText(check.name)} failed`),
1652 truncated: false,
1653 detail: null,
1654 },
1655 ),
1656 };
1657 }
1658 case "request_secret": {
1659 // T3 names a ref and a file with 32 hex characters, new for each request, in its secrets dir.
1660 const id = NodeCrypto.createHash("sha256")
1661 .update(`${sample.testCase.id}:${sample.secretRequests++}`)
1662 .digest("hex")
1663 .slice(0, 32);
1664 const home =
1665 /^\/(?:home|Users)\/[^/]+/.exec(sample.base.environment.cwd)?.[0] ?? "/home/user";
1666 const fileFor = (key: string) => `${home}/.t3/userdata/secrets/files/${key}`;
1667 // A card with `fields` saves one file per field name.
1668 if (Array.isArray(input.fields) && !pastedSecret(name, input)) {
1669 const names = input.fields.flatMap((field) =>
1670 isRecord(field) && typeof field.name === "string" ? [field.name] : [],
1671 );
1672 const fileId = (field: string) =>
1673 NodeCrypto.createHash("sha256").update(`${id}:${field}`).digest("hex").slice(0, 32);
1674 return {
1675 status: "saved",
1676 files: Object.fromEntries(names.map((field) => [field, fileFor(fileId(field))])),
1677 };
1678 }
1679 return input.delivery === "file" || pastedSecret(name, input)
1680 ? { status: "saved", file: fileFor(id) }
1681 : { status: "saved", secretRef: `secret-ref:${id}` };
1682 }
1683 case "unlink_pull_request":
1684 return { ...pr(), wasLinked: sample.linked.delete(pr().url) };
1685 case "unlink_pull_requests": {
1686 const states = Array.isArray(input.states) ? input.states : [];
1687 const unlinked = [...sample.linked]
1688 .map((url) => linkedEntry(sample, url))
1689 .filter((entry) => states.includes(entry.state));
1690 for (const { url } of unlinked) sample.linked.delete(url);
1691 const groups = Map.groupBy(unlinked, (entry) => `${entry.repository}/${entry.state}`);
1692 return {
1693 unlinked: [...groups.values()].map((entries) => ({
1694 host: "github.com",
1695 repository: entries[0]!.repository,
1696 state: entries[0]!.state,
1697 numbers: entries.map((entry) => entry.number),
1698 })),
1699 unlinkedCount: unlinked.length,
1700 remaining: sample.linked.size,
1701 };
1702 }
1703 case "watch_pull_request":
1704 sample.linked.add(pr().url);
1705 return { ...pr(), watching: true, wasWatching: false };
1706 case "unwatch_pull_request":
1707 return { ...pr(), watching: false, wasWatching: true };
1708 case "list_thread_pull_requests": {
1709 const states = Array.isArray(input.states) ? input.states : undefined;
1710 const layers =
1711 sample.testCase.setup.world?.stack === undefined
1712 ? []
1713 : worldPullRequests(sample).map((pr) => Number(pr.number));
1714 const matching = [...sample.linked]
1715 .map((url) => linkedEntry(sample, url))
1716 .map((entry) => {
1717 const position =
1718 entry.repository.toLowerCase() === sample.repo.toLowerCase()
1719 ? layers.indexOf(entry.number) + 1
1720 : 0;
1721 return {
1722 ...entry,
1723 source: "agent",
1724 watching: false,
1725 stack: position === 0 ? null : { kind: "native", position, size: layers.length },
1726 };
1727 })
1728 .filter((entry) => states === undefined || states.includes(entry.state));
1729 const start = Number(valueText(input.cursor)) || 0;
1730 const end = start + 50;
1731 const linkedLayers = layers.filter((number) =>
1732 matching.some((entry) => entry.stack !== null && entry.number === number),
1733 );
1734 return {
1735 pullRequests: matching.slice(start, end),
1736 chains: linkedLayers.length < 2 ? [] : [{ kind: "native", numbers: linkedLayers }],
1737 nextCursor: end < matching.length ? String(end) : null,
1738 };
1739 }
1740 case "t3_thread_read": {
1741 const threadId = valueText(input.threadId);
1742 const launched = sample.threads.get(threadId);
1743 if (launched !== undefined) {
1744 const items = launched.brief
1745 ? [{ position: 1, createdBy: "agent", creationSource: "mcp", text: launched.brief }]
1746 : [];
1747 const thread = {
1748 threadId,
1749 title: launched.title,
1750 createdBy: "agent",
1751 creationSource: "mcp",
1752 status: launched.brief ? "running" : "idle",
1753 };
1754 return { thread, items, nextPosition: items.length + 1, hasMore: false };
1755 }
1756 const { title, status, items } = DEFAULT_THREAD;
1757 return { thread: { threadId, title, status }, items, nextPosition: 3, hasMore: false };
1758 }
1759 case "html_preview":
1760 return { width: 728, contentHeight: 560, capturedHeight: 560, consoleMessages: [] };
1761 case "html_render":
1762 return {
1763 htmlRender: { attachmentId: "att-1", title: input.title ?? "Visual", height: 560 },
1764 message: HTML_RENDERED,
1765 };
1766 case "preview_status":
1767 return {
1768 available: false,
1769 visible: false,
1770 tabId: null,
1771 url: null,
1772 title: null,
1773 loading: false,
1774 };
1775 default:
1776 return { ok: true };
1777 }
1778 };
1779
1780 /** The machine the fake threads live on. */
1781 const EVAL_ENVIRONMENT_ID = EnvironmentId.make("eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee");
1782
1783 // The note T3 adds when a secret card ends unsaved, on refs that have one (looked up by name so
1784 // the runner still builds on refs without it).
1785 const unsavedSecretNote = (Contracts as Record<string, unknown>).SECRET_REQUEST_UNSAVED_NOTE as
1786 | Readonly<Record<string, string>>
1787 | undefined;
1788
1789 /** Fields T3 adds to real thread and task results, applied to mocks and defaults alike. */
1790 const withProductFields = (name: string, value: unknown, child: boolean): unknown => {
1791 if (!isRecord(value)) return value;
1792 // The guide is the checkout's text, so a case's mock cannot go stale.
1793 if (name === "t3_theme_read") return { ...value, guide: THEME_GUIDE };
1794 const linked = (thread: unknown, idKey = "threadId") => {
1795 if (!isRecord(thread) || thread.link !== undefined) return thread;
1796 const id = thread[idKey] ?? thread.id;
1797 return typeof id === "string"
1798 ? {
1799 ...thread,
1800 link: formatThreadLink(
1801 { environmentId: EVAL_ENVIRONMENT_ID, threadId: ThreadId.make(id) },
1802 typeof thread.title === "string" ? thread.title : undefined,
1803 ),
1804 }
1805 : thread;
1806 };
1807 const listed = (key: string) =>
1808 Array.isArray(value[key]) ? { ...value, [key]: value[key].map((item) => linked(item)) } : value;
1809 switch (name) {
1810 case "request_secret": {
1811 const note = typeof value.status === "string" ? unsavedSecretNote?.[value.status] : undefined;
1812 return note === undefined || value.note !== undefined ? value : { ...value, note };
1813 }
1814 case "delegate_task":
1815 case "task_status": {
1816 const task = linked(value, "childThreadId") as Record<string, unknown>;
1817 const running = task.status === "running" && task.next === undefined;
1818 return running ? { ...task, next: ORCHESTRATOR_MCP_RUNNING_TASK_NEXT } : task;
1819 }
1820 case "t3_thread_read":
1821 return { ...value, thread: linked(value.thread) };
1822 case "t3_thread_launch":
1823 case "t3_thread_send":
1824 case "run_scheduled_task_now":
1825 return linked(value);
1826 case "t3_thread_list":
1827 case "create_threads":
1828 return listed("threads");
1829 case "t3_thread_search":
1830 return listed("matches");
1831 case "link_pull_request":
1832 case "link_pull_requests":
1833 case "pull_request_create":
1834 return child ? { ...value, next: CHILD_PULL_REQUEST_REPORT_NEXT } : value;
1835 default:
1836 return value;
1837 }
1838 };
1839
1840 /** Cases mock a Claude caller, so the inherited fields follow the model under test. */
1841 const capabilities = ({ base }: Sample, value: unknown) =>
1842 isRecord(value)
1843 ? {
1844 ...value,
1845 inheritedModel: base.model,
1846 ...(base.codex
1847 ? {
1848 inheritedProviderInstanceId: "codex",
1849 inheritedOptions: [{ id: "reasoningEffort", value: base.codex.reasoning.effort }],
1850 }
1851 : {}),
1852 }
1853 : value;
1854
1855 /** ToolSearch: `select:a,b` loads those tools. Other queries load tools whose name has a word. */
1856 const searchTools = (base: Base, query: string, max: number) => {
1857 if (query.startsWith("select:")) {
1858 const wanted = query.slice("select:".length).split(",");
1859 return base.deferred.filter((tool) =>
1860 wanted.some((name) => [tool.name, normalizeToolName(tool.name)].includes(name.trim())),
1861 );
1862 }
1863 const words = query
1864 .toLowerCase()
1865 .split(/[\s+]+/)
1866 .filter(Boolean);
1867 const hits = (tool: ApiTool) => words.filter((word) => tool.name.toLowerCase().includes(word));
1868 const found = base.deferred.filter((tool) => hits(tool).length > 0);
1869 return found.toSorted((a, b) => hits(b).length - hits(a).length).slice(0, max);
1870 };
1871
1872 /** Tools that act on the branch's PR when no number is passed. */
1873 const BRANCH_PULL_REQUEST_TOOLS = new Set([
1874 "pull_request_read",
1875 "pull_request_update",
1876 "pull_request_respond",
1877 "pull_request_review",
1878 "pull_request_merge",
1879 "ci_logs",
1880 "ci_rerun",
1881 ]);
1882 const BRANCH_HAS_NO_PR_ERROR =
1883 "This thread's branch has no known PR. Pass number, or open one with pull_request_create.";
1884
1885 const SUBAGENT_WATCH_ERROR =
1886 "This thread is a subagent, so it cannot watch pull requests. Its parent thread owns the pull request: finish your task and report back instead.";
1887
1888 /** Result for a native tool the case does not mock, other than shell, file, and web tools. */
1889 const nativeResult = (name: string, input: Readonly<Record<string, unknown>>) => {
1890 const skill = valueText(input.skill);
1891 const results: Readonly<Record<string, string>> = {
1892 Agent: "The agent finished. Summary: the task is done and nothing else needs attention.",
1893 Skill: `Launching skill: ${skill}\n\nBase directory for this skill: ~/.claude/skills/${skill}\n\nThis skill adds no steps for this task. Do it with your usual tools.`,
1894 WebSearch: "No results found.",
1895 };
1896 return results[name] ?? "OK";
1897 };
1898
1899 const SHELL_PROMPT = `You play the computer a coding agent works on. You get a description of the machine, the agent's tool calls so far with their outputs, and one new call. Reply in JSON. In "facts", note briefly what the output must agree with. In "output", put only the call's output, exactly as the tool prints it: no notes, headings, code fences, or corrections. The agent sees "output" unchanged.
1900
1901 - Bash: what the command prints in a real terminal (git, gh, ls, cat, grep, package managers). If it fails, start with "Exit code N", then the error. If it prints nothing, reply "(Bash completed with no output)".
1902 - Read: the file with numbered lines, like \`cat -n\`. Write: "File created successfully at: <path>". Edit: "The file <path> has been updated successfully." Edit may instead supply a Codex apply_patch patch; apply its additions, deletions and hunks. A missing file, or an Edit whose old_string is not in the file: "<tool_use_error>reason</tool_use_error>".
1903 - WebFetch: the page's text.
1904
1905 Stay consistent with the machine and with earlier outputs: edits, commits, pushes, and new PRs show in later output. Where the description is silent, invent plausible small details that fit it (file contents, SHAs, test names) rather than printing nothing. Commands succeed unless the description says otherwise, including git push, gh pr create, installs, builds, and tests. A new PR gets the next number after the highest one in the description or the history, or 124 if there is none.`;
1906
1907 const ShellReply = Schema.Struct({ facts: Schema.String, output: Schema.String });
1908 const SHELL_REPLY_SCHEMA = {
1909 ...Schema.toJsonSchemaDocument(ShellReply).schema,
1910 additionalProperties: false,
1911 };
1912 const decodeShellReply = Schema.decodeUnknownOption(Schema.fromJsonString(ShellReply));
1913
1914 const callText = ({ tool, input }: Omit<ShellEntry, "output">) =>
1915 `<call>${tool} ${valueText(input)}</call>`;
1916
1917 /**
1918 * Absolute paths the case's context or files name exist, wherever the template's working
1919 * directory is. Without this the simulator denies a checkout the agent was told to work in.
1920 */
1921 export const namedPathsLine = ({ setup }: McpToolUseCase) => {
1922 const absolute = /(?:^|[\s(`'"])((?:\/home|\/Users|\/tmp)\/[^\s`'",;:)]+)/g;
1923 const fromContext = [...(setup.context ?? "").matchAll(absolute)].flatMap(([, path]) =>
1924 path ? [path] : [],
1925 );
1926 const fromFiles = Object.keys(isRecord(setup.world?.files) ? setup.world.files : {}).filter(
1927 (path) => path.startsWith("/"),
1928 );
1929 const paths = [
1930 ...new Set([...fromContext, ...fromFiles].map((path) => path.replace(/[.]+$/, ""))),
1931 ];
1932 return paths.length > 0
1933 ? [
1934 `These paths exist on this machine, with the contents the world gives: ${paths.join(", ")}. The working directory above is only where the shell starts; the agent may cd into them.`,
1935 ]
1936 : [];
1937 };
1938
1939 /** The simulator's request: the machine, this sample's earlier calls, then the new call. */
1940 export const shellRequest = (sample: Sample, call: Omit<ShellEntry, "output">, model: string) => {
1941 const { testCase, base, repo, shell, devices } = sample;
1942 const { world, context } = testCase.setup;
1943 const { cwd, platform, date } = base.environment;
1944 const command = call.tool === "Bash" && isRecord(call.input) ? call.input.command : undefined;
1945 const mocks = shellMocksFor(sample, typeof command === "string" ? command : undefined);
1946 const machine = [
1947 `Working directory: ${cwd} on ${platform}, a git checkout of ${world?.remote ?? `github.com/${repo}`}.${date ? ` Today is ${date}.` : ""}`,
1948 gitStatus(testCase),
1949 `World: ${valueText(world ?? {})}`,
1950 ...(mocks.length > 0
1951 ? [`Commands that match these regexes print exactly this output: ${valueText(mocks)}`]
1952 : []),
1953 // Which device each launcher and session drives, and which of this call's opens are refused.
1954 ...(devices?.describe(shell, typeof command === "string" ? command : undefined) ?? []),
1955 ...(world?.deviceInvalidCapture
1956 ? [
1957 "T3's device_screenshot gets each capture as a PNG whose header says 0×0, so it reports 0×0. agent-device's own screenshot is unaffected.",
1958 ]
1959 : []),
1960 ...(sample.secretFiles.length > 0
1961 ? [
1962 `T3's request_secret saved these files in this sample: ${sample.secretFiles.join(", ")}. Each exists, is readable only by the user, and holds exactly the value the user gave, typed into T3's private card or pasted in chat and saved by the agent, the right one for the task, with no trailing newline. A program that reads it, as in "$(cat <file>)", < <file> or a --*-file flag, gets that value. What appears in the output is whatever that program prints: one that prints or logs its input shows the value. No other file in that directory exists.`,
1963 ]
1964 : []),
1965 ...(sample.plainFiles.length > 0
1966 ? [
1967 `It also saved these files for card fields the agent marked not secret: ${sample.plainFiles.map(({ file, label }) => `${file} (${label})`).join(", ")}. Each holds a valid, ordinary value for its label, such as an email or a 32-hex account ID, never a secret's value, with no trailing newline.`,
1968 ]
1969 : []),
1970 ...(sample.forgeChanges.length > 0
1971 ? [
1972 `The agent made these changes through T3's tools; gh, git fetch and the host show them: ${sample.forgeChanges.join("; ")}`,
1973 ]
1974 : []),
1975 ...(context ? [`The agent was told: ${context}`] : []),
1976 `The agent's task: ${testCase.prompt}`,
1977 ...namedPathsLine(testCase),
1978 ];
1979 const history = shell.map((entry) => `${callText(entry)}\n<output>\n${entry.output}\n</output>`);
1980 return {
1981 model,
1982 max_tokens: 8_000,
1983 // The "facts" field keeps the simulator's reasoning out of the output the agent sees.
1984 output_config: { effort: "low", format: { type: "json_schema", schema: SHELL_REPLY_SCHEMA } },
1985 // A case with devices also gets agent-device's own CLI, so T3's launchers answer as it does.
1986 system: devices ? `${SHELL_PROMPT}\n\n${AGENT_DEVICE_SHELL}` : SHELL_PROMPT,
1987 messages: [
1988 {
1989 role: "user",
1990 content: [machine, [...history, `New call:\n${callText(call)}`]].map((part) =>
1991 text(part.join("\n\n")),
1992 ),
1993 },
1994 ],
1995 };
1996 };
1997
1998 /** The same simulator request in Responses form, for a GPT shell model through the proxy. */
1999 export const shellResponsesRequest = (body: ReturnType<typeof shellRequest>) => ({
2000 model: body.model,
2001 stream: true,
2002 store: false,
2003 reasoning: { effort: "low" },
2004 instructions: body.system,
2005 text: {
2006 format: { type: "json_schema", name: "shell_reply", schema: SHELL_REPLY_SCHEMA, strict: true },
2007 },
2008 input: body.messages.map((message) => ({
2009 role: message.role,
2010 content: message.content.map((block) => ({ type: "input_text", text: block.text })),
2011 })),
2012 });
2013
2014 /** The tool output in the simulator's JSON reply, or its raw text if that is not JSON. */
2015 export const shellOutput = (content: ReadonlyArray<Block>) => {
2016 const reply = content.flatMap((block) => (isText(block) ? [block.text] : [])).join("");
2017 return Option.match(decodeShellReply(reply), {
2018 onNone: () => reply.trim(),
2019 onSome: ({ output }) => output.trim(),
2020 });
2021 };
2022
2023 /**
2024 * The case's shell mocks for a command. A file for a `secret: false` field holds an ordinary
2025 * value, so a command that reads one skips the mocks that print a secret file; others still apply.
2026 */
2027 const shellMocksFor = (sample: Sample, command: string | undefined) => {
2028 const mocks = sample.testCase.setup.shellMocks ?? [];
2029 const plain =
2030 command !== undefined &&
2031 sample.plainFiles.some(({ file }) => command.includes(file.slice(-32)));
2032 return plain ? mocks.filter((mock) => !mock.match.includes("secrets/files")) : mocks;
2033 };
2034
2035 /** A shell mock's output, when the command is one simple command that the mock matches. */
2036 export const mockOutput = (sample: Sample, command: string) =>
2037 /&&|\|\||[;|\n]/.test(command.trim())
2038 ? undefined
2039 : shellMocksFor(sample, command).find((mock) => new RegExp(mock.match).test(command))?.output;
2040
2041 type Config = {
2042 readonly apiKey: string;
2043 readonly model: Model;
2044 readonly proxy?: { readonly url: string; readonly headers: Readonly<Record<string, string>> };
2045 readonly effort: string;
2046 readonly maxSteps: number;
2047 readonly shellModel: ShellModel;
2048 };
2049
2050 /**
2051 * Output for a command that needs no simulator: a matching shell mock, else, for a plain command
2052 * on a launcher T3 returned, agent-device's help or its error for a session not yet started.
2053 */
2054 export const cannedOutput = (sample: Sample, command: string) =>
2055 mockOutput(sample, command) ??
2056 launcherHelp(sample.devices?.launchers ?? [], command) ??
2057 sample.devices?.sessionError(sample.shell, command);
2058
2059 /** Output for a shell, file, or web tool: canned output, else the shell simulator. */
2060 const simulate = Effect.fn("evaluateMcpToolUse.simulate")(function* (
2061 sample: Sample,
2062 config: Config,
2063 call: Omit<ShellEntry, "output">,
2064 ) {
2065 const command = call.tool === "Bash" && isRecord(call.input) ? call.input.command : undefined;
2066 let output = typeof command === "string" ? cannedOutput(sample, command) : undefined;
2067 if (output === undefined) {
2068 const body = shellRequest(sample, call, config.shellModel);
2069 const response = isPricedModel(config.shellModel)
2070 ? yield* callModel(config.apiKey, body, config.apiKey ? undefined : config.proxy)
2071 : yield* callCodex(config, shellResponsesRequest(body));
2072 addUsage(sample.shellUsage, response.usage);
2073 output = shellOutput(response.content);
2074 }
2075 if (call.tool === "Bash" && output === "") output = "(Bash completed with no output)";
2076 const entry = { ...call, output };
2077 sample.shell.push(entry);
2078 for (const url of createdPullRequests(entry)) sample.linked.add(url);
2079 sample.openedPullRequest ??= branchPullRequestCreatedBy(
2080 entry,
2081 sample.testCase.setup.world?.branch,
2082 );
2083 return output;
2084 });
2085
2086 /** Tells the shell simulator about a file request_secret saved, so it exists for later commands. */
2087 const savedSecretFile = (
2088 sample: Sample,
2089 input: Readonly<Record<string, unknown>>,
2090 result: unknown,
2091 ) => {
2092 if (!isRecord(result)) return;
2093 if (typeof result.file === "string") sample.secretFiles.push(result.file);
2094 if (!isRecord(result.files)) return;
2095 const fields = Array.isArray(input.fields) ? input.fields.filter(isRecord) : [];
2096 for (const [name, file] of Object.entries(result.files)) {
2097 if (typeof file !== "string") continue;
2098 const field = fields.find((candidate) => candidate.name === name);
2099 if (field?.secret === false) sample.plainFiles.push({ file, label: valueText(field.label) });
2100 else sample.secretFiles.push(file);
2101 }
2102 };
2103
2104 /** The result for one tool call. */
2105 const FORGE_WRITE_TOOLS = new Set([
2106 "pull_request_create",
2107 "pull_request_update",
2108 "pull_request_respond",
2109 "pull_request_review",
2110 "pull_request_merge",
2111 "ci_rerun",
2112 ]);
2113
2114 const recordForgeChange = (
2115 sample: Sample,
2116 name: string,
2117 input: Readonly<Record<string, unknown>>,
2118 result: unknown,
2119 ) => {
2120 if (FORGE_WRITE_TOOLS.has(name)) {
2121 sample.forgeChanges.push(`${name} ${valueText(input)} returned ${valueText(result)}`);
2122 }
2123 };
2124
2125 /**
2126 * A successful `git push` in this sample published `branch`: a bare push or a push to a remote
2127 * alone pushes the current branch; otherwise a refspec has to name it or HEAD. Dry runs don't count.
2128 */
2129 const pushed = (sample: Sample, branch: string) => {
2130 // HEAD and a bare push publish whatever is checked out when that push runs.
2131 let current = sample.testCase.setup.world?.branch ?? "";
2132 for (const entry of sample.shell) {
2133 const succeeded = entry.tool === "Bash" && !/^Exit code [1-9]/.test(entry.output);
2134 for (const part of shellParts(entry)) {
2135 current = switchedTo(part) ?? current;
2136 const words = /\bgit\s+push\b(.*)/.exec(part)?.[1]?.trim().split(/\s+/).filter(Boolean);
2137 if (!succeeded || words === undefined) continue;
2138 if (["--dry-run", "-n", "--help", "-h"].some((flag) => words.includes(flag))) continue;
2139 const refs = words.filter((word) => !word.startsWith("-")).slice(1);
2140 const sources = refs.map((ref) => ref.split(":")[0]!.replace(/^\+/, ""));
2141 if (
2142 refs.length === 0
2143 ? current === branch
2144 : sources.some((source) => source === branch || (source === "HEAD" && current === branch))
2145 )
2146 return true;
2147 }
2148 }
2149 return false;
2150 };
2151
2152 /**
2153 * Refusals the real tools give for this world, so the tools arm meets the same walls as the CLI.
2154 * Messages are PullRequestService's and GitManager's.
2155 */
2156 const productRefusal = (sample: Sample, name: string, input: Readonly<Record<string, unknown>>) => {
2157 const world = sample.testCase.setup.world;
2158 if (
2159 name === "pull_request_create" &&
2160 world?.unpushed === true &&
2161 !pushed(sample, checkedOutBranch(sample))
2162 ) {
2163 return "Current branch has not been pushed. Push before creating a PR.";
2164 }
2165 if (name === "pull_request_merge" && typeof input.expectedHeadSha === "string") {
2166 const own = branchPullRequest(sample);
2167 const number = input.number ?? own.number;
2168 const pr = worldPullRequests(sample).find((candidate) => Number(candidate.number) === number);
2169 // The same head pull_request_read reports: the world's, else the checkout's for its own PR.
2170 const head =
2171 typeof pr?.headRefOid === "string"
2172 ? pr.headRefOid
2173 : number === own.number
2174 ? world?.commits?.[0]?.sha
2175 : undefined;
2176 if (head !== undefined && head !== input.expectedHeadSha.trim()) {
2177 return `The head is now ${head}, not ${input.expectedHeadSha}. Review the new commits first.`;
2178 }
2179 }
2180 return undefined;
2181 };
2182
2183 export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(function* (
2184 sample: Sample,
2185 config: Config,
2186 call: typeof ToolUse.Type,
2187 ) {
2188 const { testCase, base, tools } = sample;
2189 let name = normalizeToolName(call.name);
2190 let input = isRecord(call.input) ? call.input : {};
2191 if (call.name === "web__run") {
2192 const open = Array.isArray(input.open) ? input.open[0] : undefined;
2193 const search = Array.isArray(input.search_query) ? input.search_query[0] : undefined;
2194 name = open ? "WebFetch" : "WebSearch";
2195 input = isRecord(open)
2196 ? { url: open.ref_id, prompt: "Return the page content." }
2197 : isRecord(search)
2198 ? { query: search.q }
2199 : input;
2200 }
2201 const failed = (error: string): Block => ({ ...toolResult(call.id, error), is_error: true });
2202 if (!base.codex && !hasTool(tools, call.name)) {
2203 const error = `InputValidationError: ${call.name} is deferred. Load it with ToolSearch first.`;
2204 return failed(`<tool_use_error>${error}</tool_use_error>`);
2205 }
2206 if (name === "ToolSearch") {
2207 const max = typeof input.max_results === "number" ? input.max_results : 5;
2208 const found = searchTools(base, valueText(input.query), max);
2209 if (found.length === 0) return toolResult(call.id, "No matching deferred tools found");
2210 return loadTools(tools, call.id, found);
2211 }
2212 // T3 refuses a page call without exactly one source before any mocked result.
2213 const sources = [input.html, input.path].filter(
2214 (source) => source !== undefined && source !== null,
2215 );
2216 if (isPageTool(name) && (sources.length !== 1 || typeof sources[0] !== "string")) {
2217 return failed("Pass exactly one of html or path.");
2218 }
2219 if (isPageTool(name) && typeof input.path === "string" && !input.path.startsWith("/")) {
2220 return failed(`${input.path} is not a readable HTML document. Pass an absolute path.`);
2221 }
2222 if (
2223 name === "request_secret" &&
2224 input.fields !== undefined &&
2225 (input.value !== undefined || input.delivery === "secretRef")
2226 ) {
2227 return failed(
2228 '`fields` values are saved as files. Leave out `value` and `delivery: "secretRef"`.',
2229 );
2230 }
2231 const mock = nextMock(sample, name, testCase.setup.mocks?.[name]);
2232 if (isT3Name(call.name) && PREVIEW_TOOLS.has(name)) {
2233 if (testCase.setup.world?.browserAgentAccess === false)
2234 return failed(
2235 new PreviewAutomationUnavailableError({
2236 capability: "preview",
2237 environmentId: EVAL_ENVIRONMENT_ID,
2238 }).message,
2239 );
2240 const previewError = previewFailure(mock);
2241 if (previewError !== undefined) return failed(previewError.message);
2242 }
2243 // T3 answers device calls from the world. A mock that is a T3 device error fails inside T3.
2244 const failure = deviceFailure(mock);
2245 const fromWorld = mock === undefined || mock === "default" || failure !== undefined;
2246 if (sample.devices && isT3Name(call.name) && isDeviceTool(name) && fromWorld) {
2247 const result = yield* sample.devices.call(name, input, failure);
2248 return result.isError ? failed(result.text) : toolResult(call.id, result.text);
2249 }
2250 if (sample.backgroundWait && isT3Name(call.name) && name === "t3_background_wait" && fromWorld) {
2251 const result = yield* sample.backgroundWait(input);
2252 return result.isError ? failed(result.text) : toolResult(call.id, result.text);
2253 }
2254 // `{"$error": "..."}` fails the call with that text, as T3 reports a forge error.
2255 if (isRecord(mock) && typeof mock.$error === "string") return failed(mock.$error);
2256 if (isT3Name(call.name)) {
2257 const refused = productRefusal(sample, name, input);
2258 if (refused !== undefined) return failed(refused);
2259 }
2260 // A pasted secret is saved without a card, so a mocked card outcome does not apply.
2261 if (mock !== undefined && mock !== "default" && !pastedSecret(name, input)) {
2262 if (name === "request_secret") savedSecretFile(sample, input, mock);
2263 if (name === "pull_request_create" && isRecord(mock) && typeof mock.url === "string") {
2264 sample.linked.add(mock.url);
2265 const number = Number(mock.number);
2266 if (number > 0) sample.openedPullRequest = number;
2267 }
2268 recordForgeChange(sample, name, input, mock);
2269 return toolResult(
2270 call.id,
2271 name === "orchestrator_capabilities"
2272 ? capabilities(sample, mock)
2273 : withProductFields(name, mock, testCase.setup.role === "delegated-child"),
2274 );
2275 }
2276 if (isT3Name(call.name)) {
2277 // T3 refuses a watch from a delegated child.
2278 if (name === "watch_pull_request" && testCase.setup.role === "delegated-child") {
2279 return failed(SUBAGENT_WATCH_ERROR);
2280 }
2281 if (BRANCH_PULL_REQUEST_TOOLS.has(name) && input.number === undefined) {
2282 // Until the agent opens it, the PR a create would open does not exist.
2283 if (branchPullRequest(sample).created) return failed(BRANCH_HAS_NO_PR_ERROR);
2284 }
2285 if (name === "ci_logs" || name === "ci_rerun") {
2286 const { all, checks, named } = selectedWorldChecks(sample, input);
2287 if (named !== null && checks.length === 0) {
2288 const names = all.map((check) => valueText(check.name)).join(", ") || "none";
2289 return failed(`No check is named ${valueText(input.check)}. Checks: ${names}.`);
2290 }
2291 }
2292 const value = t3Default(sample, name, input);
2293 if (name === "request_secret") savedSecretFile(sample, input, value);
2294 recordForgeChange(sample, name, input, value);
2295 return toolResult(
2296 call.id,
2297 name === "orchestrator_capabilities"
2298 ? capabilities(sample, value)
2299 : withProductFields(name, value, testCase.setup.role === "delegated-child"),
2300 );
2301 }
2302 if (call.name === "clock__curr_time")
2303 return toolResult(call.id, {
2304 current_time: `${base.environment.date ?? "2026-01-01"} 12:00:00 UTC`,
2305 });
2306 if (!SHELL_TOOLS.has(name)) return toolResult(call.id, nativeResult(name, input));
2307 const output = yield* simulate(sample, config, { tool: name, input });
2308 // Claude Code marks a failed command or file call as an error.
2309 return /^(Exit code [1-9]|<tool_use_error>)/.test(output)
2310 ? failed(output)
2311 : toolResult(call.id, output);
2312 });
2313
2314 const BOUND = /^([<>]=)(-?\d+(?:\.\d+)?)$/;
2315 /**
2316 * Partial match: `~text` is a case-insensitive substring, `<=N` / `>=N` bound a number, and `null`
2317 * matches a missing or null value.
2318 */
2319 const matches = (expected: unknown, actual: unknown): boolean => {
2320 if (expected === null) return actual === undefined || actual === null;
2321 const bound = typeof expected === "string" ? BOUND.exec(expected) : null;
2322 if (bound !== null) {
2323 const limit = Number(bound[2]);
2324 return typeof actual === "number" && (bound[1] === "<=" ? actual <= limit : actual >= limit);
2325 }
2326 if (typeof expected === "string" && expected.startsWith("~")) {
2327 return valueText(actual).toLowerCase().includes(expected.slice(1).toLowerCase());
2328 }
2329 if (typeof expected === "string" && expected.startsWith("re:")) {
2330 return new RegExp(expected.slice(3)).test(valueText(actual));
2331 }
2332 if (Array.isArray(expected)) {
2333 return (
2334 Array.isArray(actual) && expected.every((item) => actual.some((at) => matches(item, at)))
2335 );
2336 }
2337 if (!isRecord(expected)) return expected === actual;
2338 const at = (path: string) =>
2339 path
2340 .split(".")
2341 .reduce<unknown>((value, key) => (isRecord(value) ? value[key] : undefined), actual);
2342 return Object.entries(expected).every(([path, value]) =>
2343 // Each expected field needs its own field, so one field cannot hold every value.
2344 path === "fields" && Array.isArray(value)
2345 ? matchesEach(value, at(path))
2346 : matches(value, at(path)),
2347 );
2348 };
2349
2350 const matchesEach = (
2351 expected: ReadonlyArray<unknown>,
2352 actual: unknown,
2353 taken: ReadonlySet<number> = new Set(),
2354 ): boolean => {
2355 if (expected.length === 0) return true;
2356 const [first, ...rest] = expected;
2357 return (
2358 Array.isArray(actual) &&
2359 actual.some(
2360 (item, i) =>
2361 !taken.has(i) && matches(first, item) && matchesEach(rest, actual, new Set([...taken, i])),
2362 )
2363 );
2364 };
2365
2366 /** Each new top-level thread request in a run; a failed call created nothing. */
2367 const newThreadRequests = (calls: ReadonlyArray<Call>) =>
2368 calls.flatMap((call): ReadonlyArray<{ request: unknown; brief: unknown }> => {
2369 if (call.failed === true || !isRecord(call.input)) return [];
2370 if (call.tool === "t3_thread_launch")
2371 return [{ request: call.input, brief: call.input.message }];
2372 if (call.tool !== "create_threads" || !Array.isArray(call.input.threads)) return [];
2373 return call.input.threads.map((request: unknown) => ({
2374 request,
2375 brief: isRecord(request) ? request.prompt : undefined,
2376 }));
2377 });
2378
2379 /** Whether every expected value can take a different request it matches. */
2380 const matchesDistinct = (
2381 expected: ReadonlyArray<unknown>,
2382 requests: ReturnType<typeof newThreadRequests>,
2383 taken: ReadonlySet<number> = new Set(),
2384 ): boolean => {
2385 const [first, ...rest] = expected;
2386 if (expected.length === 0) return true;
2387 return requests.some(
2388 ({ request, brief }, i) =>
2389 !taken.has(i) &&
2390 matches(first, typeof first === "string" ? brief : request) &&
2391 matchesDistinct(rest, requests, new Set([...taken, i])),
2392 );
2393 };
2394
2395 /** delegate_task takes `target.options` as an object or `[{id, value}]`. Grade both as an object. */
2396 const withOptionsObject = (input: unknown) => {
2397 if (!isRecord(input) || !isRecord(input.target) || !Array.isArray(input.target.options)) {
2398 return input;
2399 }
2400 const options = Object.fromEntries(
2401 input.target.options.flatMap((option) =>
2402 isRecord(option) ? [[valueText(option.id), option.value]] : [],
2403 ),
2404 );
2405 return { ...input, target: { ...input.target, options } };
2406 };
2407
2408 /** What the user wrote in the case's thread: its context, prior user turns and the prompt. */
2409 /** `world.files` keys: paths relative to the thread cwd, or absolute. */
2410 export const worldFilePaths = ({ setup }: Pick<McpToolUseCase, "setup">) =>
2411 Object.keys(isRecord(setup.world?.files) ? setup.world.files : {});
2412
2413 export const pastedText = ({ prompt, setup }: Pick<McpToolUseCase, "prompt" | "setup">) =>
2414 [
2415 setup.context ?? "",
2416 ...(setup.priorTurns ?? []).flatMap((turn) => (turn.role === "user" ? [turn.text ?? ""] : [])),
2417 prompt,
2418 ].join("\n");
2419
2420 const isPageTool = (tool: string) => tool === "html_preview" || tool === "html_render";
2421
2422 /** `page` is the HTML a page tool shows: its inline `html`, or the file the agent wrote at `path`. */
2423 const withPage = (tool: string, input: unknown, files: ReadonlyMap<string, string>) => {
2424 if (!isPageTool(tool) || !isRecord(input)) return input;
2425 const path = valueText(input.path);
2426 // A patch may name the file relative to the agent's working directory. When written files that
2427 // could be this one differ, the page is unknown, so page checks fail rather than grade a stale copy.
2428 const written = [...files].filter(
2429 ([file]) =>
2430 file === path ||
2431 (!file.startsWith("/") && path.endsWith(`/${file.replace(/^(?:\.\/)+/, "")}`)),
2432 );
2433 const pages = new Set(written.map(([, page]) => page));
2434 const page =
2435 typeof input.html === "string"
2436 ? input.html
2437 : pages.size === 1 && !pages.has(UNKNOWN_SCRIPT)
2438 ? written[0]![1]
2439 : undefined;
2440 // The grader derives `page`; a call cannot supply its own.
2441 return { ...input, page };
2442 };
2443
2444 /** `pasted` is the thread's user text (`pastedText`), for request_secret's `value`. */
2445 export const gradeSample = (
2446 expect: McpToolUseCase["expect"],
2447 run: SampleRun,
2448 pasted = "",
2449 worldFiles: ReadonlyArray<string> = [],
2450 ) => {
2451 const checks: Array<{ readonly check: string; readonly pass: boolean }> = [];
2452 const add = (check: string, pass: boolean) => checks.push({ check, pass });
2453 const written = scriptsBefore(run.calls, true);
2454 // A refused page call showed nothing, so it neither previews nor renders, and no file stands in
2455 // for its page. Inline HTML it sent still counts against callsMustNot.
2456 const attempted = run.calls.map((call, index) => ({
2457 ...call,
2458 input: withPage(
2459 call.tool,
2460 withOptionsObject(call.input),
2461 call.failed ? new Map() : written[index]!,
2462 ),
2463 }));
2464 const calls = attempted.filter((call) => !(call.failed && isPageTool(call.tool)));
2465 // `tool` may list alternatives, as in "link_pull_request|watch_pull_request"; counts add up.
2466 const count = (tool: string) =>
2467 calls.filter((call) => tool.split("|").includes(call.tool)).length;
2468 const found = ({ tool, args = {}, argsAnyOf, success }: McpToolUseCall, among = calls) =>
2469 among.some(
2470 (call) =>
2471 tool.split("|").includes(call.tool) &&
2472 (success === undefined || call.failed === false) &&
2473 matches(args, call.input) &&
2474 (argsAnyOf === undefined ||
2475 argsAnyOf.some((alternative) => matches(alternative, call.input))),
2476 );
2477 const label = ({ tool, args }: McpToolUseCall) => (args ? `${tool} ${valueText(args)}` : tool);
2478 for (const call of expect.calls ?? []) add(`calls ${label(call)}`, found(call));
2479 if (expect.distinctCalls !== undefined) {
2480 const expected = expect.distinctCalls.map(({ tool, args }) => ({ tool, input: args ?? {} }));
2481 add(
2482 "distinctCalls",
2483 calls.every((call) => expected.filter((entry) => matches(entry, call)).length <= 1) &&
2484 matchesDistinct(
2485 expected,
2486 calls.map((request) => ({ request, brief: "" })),
2487 ),
2488 );
2489 }
2490 if (expect.anyOf !== undefined) {
2491 const passes = (alternative: (typeof expect.anyOf)[number]) =>
2492 gradeSample(alternative, run, pasted, worldFiles).every((check) => check.pass);
2493 add("anyOf", expect.anyOf.some(passes));
2494 }
2495 for (const call of expect.callsMustNot ?? [])
2496 add(`callsMustNot ${label(call)}`, !found(call, attempted));
2497 // A page check must not pass because the grader never saw the file a path call shows.
2498 const pageTools = [...(expect.calls ?? []), ...(expect.callsMustNot ?? [])].flatMap((call) =>
2499 call.args?.page === undefined ? [] : call.tool.split("|"),
2500 );
2501 for (const call of calls)
2502 if (
2503 pageTools.includes(call.tool) &&
2504 isRecord(call.input) &&
2505 call.input.path !== undefined &&
2506 call.input.page === undefined
2507 )
2508 add(`page known ${call.tool} ${valueText(call.input.path)}`, false);
2509 for (const tool of expect.notCalls ?? [])
2510 add(`notCalls ${tool}`, !attempted.some((call) => call.tool === tool));
2511 if (expect.order !== undefined) {
2512 const firsts = expect.order.map((tool) => calls.findIndex((call) => call.tool === tool));
2513 const inOrder = firsts.every((at, i) => at > (firsts[i - 1] ?? -1));
2514 add(`order ${expect.order.join(" > ")}`, inOrder);
2515 }
2516 for (const [t, n] of Object.entries(expect.minCalls ?? {}))
2517 add(`minCalls ${t} >= ${n}`, count(t) >= n);
2518 for (const [t, n] of Object.entries(expect.maxCalls ?? {}))
2519 add(`maxCalls ${t} <= ${n}`, count(t) <= n);
2520 if (expect.newThreads !== undefined) {
2521 const requests = newThreadRequests(calls);
2522 add(
2523 `newThreads ${valueText(expect.newThreads)}`,
2524 requests.length === expect.newThreads.length && matchesDistinct(expect.newThreads, requests),
2525 );
2526 }
2527 if (expect.endsTurnAfter !== undefined) {
2528 const lastStep = Math.max(-1, ...calls.map((call) => call.step));
2529 const last = calls.filter((call) => call.step === lastStep).map((call) => call.tool);
2530 const ended = run.stopReason === "end_turn" && last.includes(expect.endsTurnAfter);
2531 add(`endsTurnAfter ${expect.endsTurnAfter}`, ended);
2532 }
2533 // Bash and Monitor both take a shell `command`. Shell checks also read, one by one, the scripts
2534 // the agent wrote and a command ran: `node run.mjs signup.mjs` after a Write of signup.mjs.
2535 const sources = scriptSources(calls);
2536 const shell = calls.flatMap((call, i) =>
2537 isRecord(call.input) && typeof call.input.command === "string"
2538 ? [call.input.command, ...(sources[i] ?? [])]
2539 : [],
2540 );
2541 const test = (pattern: string, value: string) => new RegExp(pattern).test(value);
2542 for (const p of expect.shellMustNot ?? [])
2543 add(`shellMustNot /${p}/`, !shell.some((c) => test(p, c)));
2544 // A positive check also reads argv lists as command lines: spawnSync('pnpm', ['exec', …]).
2545 for (const p of expect.shellMust ?? [])
2546 add(
2547 `shellMust /${p}/`,
2548 shell.some((c) => test(p, c) || test(p, argvForm(c))),
2549 );
2550 for (const p of expect.textMust ?? []) add(`textMust /${p}/`, test(p, run.text));
2551 for (const p of expect.textMustNot ?? []) add(`textMustNot /${p}/`, !test(p, run.text));
2552 for (const p of expect.finalTextMust ?? [])
2553 add(`finalTextMust /${p}/`, run.stopReason === "end_turn" && test(p, run.finalText ?? ""));
2554 const isLinked = (url: string) => (run.linked ?? []).some((at) => matches(url, at));
2555 for (const url of expect.linked ?? []) add(`linked ${url}`, isLinked(url));
2556 if (expect.linkedNumbers !== undefined) {
2557 const numbers = (run.linked ?? []).map((url) => Number(/(\d+)\/?$/.exec(url)?.[1]));
2558 const exact = valueText(numbers.toSorted((a, b) => a - b));
2559 add("linkedNumbers", exact === valueText(expect.linkedNumbers.toSorted((a, b) => a - b)));
2560 }
2561 checks.push(...gradeDevice(expect.device, run));
2562 checks.push(
2563 ...gradeFileLinks(
2564 expect.fileLinks,
2565 run.stopReason === "end_turn" ? run.finalText : undefined,
2566 run.cwd,
2567 worldFiles,
2568 ),
2569 );
2570 checks.push(...gradeSecret(expect.secret, run, pasted));
2571 return checks;
2572 };
2573
2574 // Checks that can only go from pass to fail as a sample goes on.
2575 const SETTLED =
2576 /^(?:(?:notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot) |secret\.(?:chatAsk|askTools|values|fileUse|noCardDemand|afterDecline) )/;
2577 /** True once the sample has failed for good, so the rest of it cannot change the result. */
2578 export const failedEarly = (expect: McpToolUseCase["expect"], run: SampleRun, pasted = "") =>
2579 gradeSample(
2580 expect,
2581 // A page call T3 has not answered yet may still be refused, so it is graded once it settles.
2582 {
2583 ...run,
2584 calls: run.calls.filter((call) => !isPageTool(call.tool) || call.failed !== undefined),
2585 },
2586 pasted,
2587 ).some((check) => !check.pass && SETTLED.test(check.check));
2588
2589 const requestBody = ({ base, tools }: Sample, effort: string, messages: unknown) => ({
2590 model: base.model,
2591 max_tokens: 16_000,
2592 thinking: { type: "adaptive", display: "summarized" },
2593 output_config: { effort },
2594 cache_control: { type: "ephemeral" },
2595 // Like Claude Code, cache the shared system prompt so every sample reuses it.
2596 system: base.system.map((block, i) =>
2597 i === base.system.length - 1 ? { ...block, cache_control: { type: "ephemeral" } } : block,
2598 ),
2599 tools,
2600 messages,
2601 });
2602
2603 const codexRequest = (base: Base, input: ReadonlyArray<Block>) => ({
2604 model: base.model,
2605 stream: true,
2606 store: false,
2607 parallel_tool_calls: true,
2608 reasoning: base.codex?.reasoning,
2609 instructions: base.codex?.instructions,
2610 text: base.codex?.text,
2611 tool_choice: base.codex?.tool_choice,
2612 include: ["reasoning.encrypted_content"],
2613 input: [...(base.codex?.input ?? []), ...input],
2614 });
2615
2616 /** The bridge only accepts serialized data. Model code never receives host objects/functions. */
2617 export const executeCodex = async (
2618 code: string,
2619 catalog: ReadonlyArray<{ name: string; description: string }>,
2620 answer: (name: string, input: unknown) => Promise<unknown>,
2621 saved: Record<string, unknown> = {},
2622 ) => {
2623 const pending: Array<{ name: string; input: string; resolve: (value: string) => void }> = [];
2624 const bridge = Object.setPrototypeOf(
2625 (name: string, input: string, resolve: (value: string) => void) =>
2626 pending.push({ name, input, resolve }),
2627 null,
2628 );
2629 const context = NodeVM.createContext(
2630 { __bridge: bridge, __catalog: JSON.stringify(catalog), __storeJson: JSON.stringify(saved) },
2631 { codeGeneration: { strings: false, wasm: false }, microtaskMode: "afterEvaluate" },
2632 );
2633 const run = (source: string) => NodeVM.runInContext(source, context, { timeout: 250 });
2634 run(`((bridge, catalog) => {
2635 delete globalThis.__bridge; delete globalThis.__catalog;
2636 globalThis.ALL_TOOLS = catalog;
2637 globalThis.tools = Object.fromEntries(catalog.map(({name}) => [name, (input) => new Promise(resolve => bridge(name, JSON.stringify(input ?? {}), value => resolve(JSON.parse(value))))]));
2638 globalThis.__output = []; globalThis.__done = false;
2639 globalThis.text = globalThis.notify = value => __output.push(typeof value === 'string' ? value : JSON.stringify(value));
2640 globalThis.__stored = JSON.parse(__storeJson); delete globalThis.__storeJson;
2641 globalThis.store = (k,v) => __stored[k] = v; globalThis.load = k => __stored[k];
2642 })(__bridge, JSON.parse(__catalog));
2643 (async () => { ${code}\n })().then(() => { __done = true; }, e => { text(String(e)); __done = true; });`);
2644 // Unawaited calls still run, as they do in Codex, so they are answered and graded too.
2645 for (let count = 0; !context.__done || pending.length > 0; count++) {
2646 if (count === 128) throw new Error("Code cell exceeded 128 tool calls.");
2647 const job = pending.shift();
2648 if (!job) throw new Error("Code cell stalled without a tool call.");
2649 const result = await answer(job.name, JSON.parse(job.input));
2650 context.__resolve = job.resolve;
2651 context.__result = JSON.stringify(result ?? null);
2652 run("__resolve(__result); delete globalThis.__resolve; delete globalThis.__result;");
2653 }
2654 Object.assign(saved, JSON.parse(run("JSON.stringify(__stored)")));
2655 return valueText(context.__output.join("\n"));
2656 };
2657
2658 const codexCatalog = (base: Base) => [
2659 ...base.t3.map((tool) => ({
2660 name: tool.name.replace("mcp__t3-code__", "mcp__t3_code__"),
2661 description: `${valueText(tool.description)}\nInput JSON schema: ${JSON.stringify(tool.input_schema)}`,
2662 })),
2663 ...[
2664 ...valueText(base.nativeTools.find((tool) => tool.name === "exec")?.description).matchAll(
2665 /### `([^`]+)`/g,
2666 ),
2667 ].map((match) => ({ name: match[1]!, description: match[1]! })),
2668 ];
2669
2670 const CodexResponse = Schema.Struct({
2671 output: Schema.Array(AnyBlock),
2672 usage: Schema.Struct({
2673 input_tokens: Schema.Number,
2674 output_tokens: Schema.Number,
2675 input_tokens_details: Schema.Struct({ cached_tokens: Schema.Number }),
2676 }),
2677 });
2678
2679 const decodeCodexResponse = Schema.decodeUnknownEffect(CodexResponse);
2680 const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown));
2681
2682 const callCodex = Effect.fn("evaluateMcpToolUse.callCodex")(function* (
2683 config: Config,
2684 body: unknown,
2685 ) {
2686 const raw = yield* postModel("", body, config.proxy, true);
2687 const completed = raw
2688 .split("\n")
2689 .filter((line) => line.startsWith("data: "))
2690 .map((line) => JSON.parse(line.slice(6)) as Record<string, unknown>)
2691 .find((event) => event.type === "response.completed");
2692 if (!completed)
2693 return yield* Effect.die(new Error("Responses stream ended without response.completed."));
2694 const response = yield* decodeCodexResponse(completed.response);
2695 return {
2696 raw: response.output,
2697 content: response.output.flatMap((item): Array<Block> => {
2698 if (item.type === "message" && Array.isArray(item.content))
2699 return item.content.map((part: Block) => text(valueText(part.text)));
2700 if (item.type === "function_call" || item.type === "custom_tool_call")
2701 return [
2702 {
2703 type: "tool_use",
2704 id: item.call_id,
2705 name: item.name,
2706 input:
2707 item.type === "custom_tool_call" ? item.input : JSON.parse(valueText(item.arguments)),
2708 },
2709 ];
2710 return [];
2711 }),
2712 stop_reason: response.output.some(
2713 (item) => item.type.endsWith("tool_call") || item.type === "function_call",
2714 )
2715 ? "tool_use"
2716 : "end_turn",
2717 usage: {
2718 ...response.usage,
2719 input_tokens: response.usage.input_tokens - response.usage.input_tokens_details.cached_tokens,
2720 cache_read_input_tokens: response.usage.input_tokens_details.cached_tokens,
2721 },
2722 };
2723 });
2724
2725 // Concurrent runs share an organization's rate limits. Backing off for up to about 4 minutes
2726 // lets a sample wait out a busy spell, with jitter so samples do not retry in step.
2727 const RETRY = Schedule.min([Schedule.exponential("2 seconds"), Schedule.spaced("1 minute")]).pipe(
2728 Schedule.jittered,
2729 );
2730
2731 const postModel = Effect.fn("evaluateMcpToolUse.postModel")(function* (
2732 apiKey: string,
2733 body: unknown,
2734 proxy?: Config["proxy"],
2735 responses = false,
2736 ) {
2737 const client = (yield* HttpClient.HttpClient).pipe(
2738 // Anthropic answers 529 when it is overloaded; retryTransient already retries 429 and 5xx.
2739 HttpClient.filterStatus((status) => status !== 529),
2740 HttpClient.retryTransient({
2741 retryOn: "errors-and-responses",
2742 while: (error) =>
2743 error.reason._tag === "StatusCodeError" && error.reason.response.status === 529,
2744 schedule: RETRY,
2745 times: 8,
2746 }),
2747 );
2748 const response = yield* HttpClientRequest.post(
2749 `${proxy?.url ?? "https://api.anthropic.com/v1"}/${responses ? "responses" : "messages"}`,
2750 ).pipe(
2751 HttpClientRequest.setHeaders({
2752 ...(apiKey ? { "x-api-key": apiKey } : {}),
2753 "anthropic-version": "2023-06-01",
2754 // Claude Code sends this beta for `defer_loading` tools and `tool_reference` results.
2755 "anthropic-beta": "advanced-tool-use-2025-11-20",
2756 ...proxy?.headers,
2757 }),
2758 HttpClientRequest.bodyJson(body),
2759 Effect.flatMap(client.execute),
2760 );
2761 if (response.status !== 200) {
2762 const body = yield* response.text;
2763 return yield* Effect.die(new Error(`Model API ${response.status}: ${body}`));
2764 }
2765 return yield* response.text;
2766 });
2767 const callModel = (apiKey: string, body: unknown, proxy?: Config["proxy"]) =>
2768 postModel(apiKey, body, proxy).pipe(
2769 Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(ApiResponse))),
2770 );
2771
2772 /** A tool call as graded. Claude Code runs an `Agent` in the background unless told otherwise. */
2773 const recordedCall = (step: number, use: typeof ToolUse.Type): Call => {
2774 const tool = normalizeToolName(use.name);
2775 const input = normalizeToolInput(use.name, use.input);
2776 const claudeAgent = tool === "Agent" && !use.name.endsWith("spawn_agent") && isRecord(input);
2777 return {
2778 step,
2779 tool,
2780 input: claudeAgent ? { run_in_background: true, ...input } : input,
2781 id: use.id,
2782 };
2783 };
2784
2785 /** Records on a call what the agent saw, so graders can count failed attempts. */
2786 const settle = (call: Call | undefined, result: Block) => {
2787 if (call !== undefined) {
2788 call.failed = result.is_error === true;
2789 call.output = valueText(result.content);
2790 }
2791 return result;
2792 };
2793
2794 /** Calls that ran. Claude Code rejects a deferred tool that was not loaded, so it never runs. */
2795 export const ranCalls = (
2796 { base, tools }: Sample,
2797 step: number,
2798 uses: ReadonlyArray<typeof ToolUse.Type>,
2799 ) =>
2800 uses
2801 .filter((use) => base.codex !== undefined || hasTool(tools, use.name))
2802 .map((use) => recordedCall(step, use));
2803
2804 /**
2805 * Runs one sample until the model ends its turn, uses up its steps, or fails a check that no
2806 * later step can undo. Tokens add up in `spent`, so a sample that errors still reports them.
2807 */
2808 const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* (
2809 testCase: McpToolUseCase,
2810 config: Config,
2811 base: Base,
2812 spent: { readonly usage: Usage; readonly shellUsage: Usage },
2813 ) {
2814 const sample = { ...makeSample(testCase, base), shellUsage: spent.shellUsage };
2815 yield* withDevices(sample);
2816 yield* withBackgroundWait(sample);
2817 const runPromise = Effect.runPromiseWith(yield* Effect.context<HttpClient.HttpClient>());
2818 const messages = buildMessages(sample);
2819 const input = base.codex ? codexMessages(messages) : undefined;
2820 const transcript: Array<unknown> = [];
2821 const start = messages.length;
2822 const calls: Array<Call> = [];
2823 const said: Array<string> = [];
2824 const { usage } = spent;
2825 let stopReason = "max_steps";
2826 let finalText = "";
2827 // Codex's code-mode cells take more model calls, so a case's own budget grows for it too.
2828 const extra = isCodex(config.model) ? 4 : 0;
2829 const maxSteps = testCase.maxSteps === undefined ? config.maxSteps : testCase.maxSteps + extra;
2830 let modelCalls = 0;
2831 for (let step = 0; step < maxSteps; step++) {
2832 modelCalls++;
2833 const response = input
2834 ? yield* callCodex(config, codexRequest(base, input))
2835 : yield* callModel(config.apiKey, requestBody(sample, config.effort, messages));
2836 addUsage(usage, response.usage);
2837 if ("raw" in response) {
2838 input?.push(...response.raw);
2839 transcript.push(...response.raw);
2840 }
2841 messages.push({ role: "assistant", content: response.content });
2842 const texts = response.content.filter(isText).map((block) => block.text);
2843 said.push(...texts);
2844 const uses = response.content.filter(isToolUse);
2845 calls.push(...ranCalls(sample, step, uses));
2846 if (response.stop_reason !== "tool_use") {
2847 stopReason = response.stop_reason ?? "unknown";
2848 finalText = texts.join("\n\n");
2849 break;
2850 }
2851 if (
2852 failedEarly(
2853 testCase.expect,
2854 { calls, text: said.join("\n\n"), stopReason },
2855 pastedText(testCase),
2856 )
2857 ) {
2858 stopReason = "failed_early";
2859 break;
2860 }
2861 const results = yield* Effect.forEach(uses, (use) =>
2862 Effect.gen(function* () {
2863 const own = calls.find((call) => call.id === use.id);
2864 if (!input) {
2865 return settle(own, yield* answerToolCall(sample, config, use));
2866 }
2867 const answer = (name: string, args: unknown, record = true) => {
2868 const tool = normalizeToolName(name);
2869 const normalized = normalizeToolInput(name, args);
2870 // A code cell's calls are recorded here; a direct function call already was.
2871 let recorded = own;
2872 if (record) {
2873 recorded = { step, tool, input: normalized };
2874 calls.push(recorded);
2875 }
2876 const call = { ...use, name, input: normalized };
2877 return answerToolCall(sample, config, call).pipe(
2878 Effect.map((result) => settle(recorded, result)),
2879 Effect.map((result) => {
2880 const output = valueText(result.content);
2881 if (isT3Name(name))
2882 return {
2883 content: [text(output)],
2884 ...(!result.is_error
2885 ? { structuredContent: Option.getOrElse(decodeJson(output), () => output) }
2886 : { isError: true }),
2887 };
2888 return tool === "Bash"
2889 ? {
2890 output,
2891 exit_code: /^Exit code [1-9]/.test(output) ? 1 : 0,
2892 wall_time_seconds: 0,
2893 }
2894 : output;
2895 }),
2896 runPromise,
2897 );
2898 };
2899 return yield* Effect.tryPromise({
2900 try: async () => {
2901 const output =
2902 use.name === "exec"
2903 ? await executeCodex(
2904 valueText(use.input),
2905 codexCatalog(base),
2906 answer,
2907 sample.codeStore,
2908 )
2909 : await answer(use.name, use.input, false);
2910 const item = {
2911 type: use.name === "exec" ? "custom_tool_call_output" : "function_call_output",
2912 call_id: use.id,
2913 output: valueText(output),
2914 };
2915 input.push(item);
2916 transcript.push(item);
2917 return toolResult(use.id, output);
2918 },
2919 catch: (cause) => new EvalToolCallError({ tool: use.name, cause }),
2920 });
2921 }),
2922 );
2923 // Claude Code adds this text after a ToolSearch that loaded tools.
2924 const loaded = results.some((result) => Array.isArray(result.content));
2925 messages.push({ role: "user", content: [...results, ...(loaded ? [text(TOOL_LOADED)] : [])] });
2926 if (
2927 failedEarly(
2928 testCase.expect,
2929 { calls, text: said.join("\n\n"), stopReason },
2930 pastedText(testCase),
2931 )
2932 ) {
2933 stopReason = "failed_early";
2934 break;
2935 }
2936 }
2937 // The transcript (model turns and tool results) shows what the agent saw when a case fails.
2938 return {
2939 calls,
2940 text: said.join("\n\n"),
2941 stopReason,
2942 finalText,
2943 modelCalls,
2944 cwd: base.environment.cwd,
2945 linked: [...sample.linked],
2946 ...(sample.devices ? { launchers: [...sample.devices.launchers] } : {}),
2947 usage,
2948 shellUsage: sample.shellUsage,
2949 transcript: input ? transcript : messages.slice(start),
2950 };
2951 }, Effect.scoped);
2952
2953 /**
2954 * Model requests a sample made. Rows from before `modelCalls` was recorded count up to the last
2955 * step with tool calls, plus the final answer's request when the turn ended.
2956 */
2957 export const modelRequests = (row: {
2958 readonly modelCalls?: number | undefined;
2959 readonly calls?: ReadonlyArray<{ readonly step: number }> | undefined;
2960 readonly stopReason?: string | undefined;
2961 }) => {
2962 if (row.modelCalls !== undefined) return row.modelCalls;
2963 const calls = row.calls ?? [];
2964 if (calls.length === 0) return 1;
2965 return Math.max(...calls.map((call) => call.step)) + 1 + (row.stopReason === "end_turn" ? 1 : 0);
2966 };
2967
2968 type ResultRow = Pick<McpToolUseCase, "id" | "kind" | "pattern"> & {
2969 readonly role: string;
2970 readonly pass: boolean;
2971 /** Set when the sample could not run to the end, even after retries. It is not graded. */
2972 readonly error?: string;
2973 readonly usage: Usage;
2974 readonly shellUsage: Usage;
2975 };
2976
2977 /** The share of samples that passed. Samples that errored are left out: NaN if all did. */
2978 const passRate = (
2979 rows: ReadonlyArray<{ readonly pass: boolean; readonly error?: string | undefined }>,
2980 ) => {
2981 const finished = rows.filter((row) => row.error === undefined);
2982 return finished.filter((row) => row.pass).length / finished.length;
2983 };
2984 const percent = (rate: number) => (Number.isNaN(rate) ? "-" : `${Math.round(rate * 100)}%`);
2985 const groups = <A>(rows: ReadonlyArray<A>, key: (row: A) => string) =>
2986 [...Map.groupBy(rows, key)].toSorted(([left], [right]) => left.localeCompare(right));
2987
2988 const USAGE_KEYS = ["input", "output", "cacheWrite", "cacheRead"] as const;
2989 /** Token totals and their cost in USD at one model's prices. Proxy runs have no price. */
2990 const spend = (usages: ReadonlyArray<Usage>, model: ShellModel | undefined) => {
2991 const total = (key: keyof Usage) => usages.reduce((sum, usage) => sum + usage[key], 0);
2992 const price = (key: keyof Usage) => (model && isPricedModel(model) ? PRICES[model][key] : 0);
2993 const cost = USAGE_KEYS.reduce((sum, key) => sum + total(key) * price(key), 0) / 1e6;
2994 return { cost, tokens: USAGE_KEYS.map((key) => `${key} ${total(key)}`).join(", ") };
2995 };
2996
2997 /** Markdown report. With a baseline, each case shows its earlier pass rate and the change. */
2998 export const summarize = (
2999 rows: Array<ResultRow>,
3000 options: { readonly header: string; readonly shellModel: ShellModel; readonly model: Model },
3001 before?: Map<string, number>,
3002 ) => {
3003 const codex = isCodex(options.model);
3004 const agent = spend(
3005 rows.map((row) => row.usage),
3006 codex ? undefined : options.model,
3007 );
3008 const shell = spend(
3009 rows.map((row) => row.shellUsage),
3010 options.shellModel,
3011 );
3012 const cost = agent.cost + shell.cost;
3013 const table = (title: string, key: (row: ResultRow) => string, compare = false) => [
3014 `## ${title}\n`,
3015 compare ? "| Group | Pass | Baseline | Delta |" : "| Group | Pass |",
3016 compare ? "| --- | --- | --- | --- |" : "| --- | --- |",
3017 ...groups(rows, key).map(([group, members]) => {
3018 const now = passRate(members);
3019 const was = before?.get(group);
3020 const errored = members.filter((row) => row.error !== undefined).length;
3021 const delta =
3022 was === undefined
3023 ? "new | |"
3024 : `${percent(was)} | ${Number.isNaN(now) ? "" : `${Math.round((now - was) * 100)} pts`} |`;
3025 const pass = `${percent(now)} of ${members.length - errored}${errored > 0 ? `, ${errored} errored` : ""}`;
3026 return `| ${group} | ${pass} |${compare ? ` ${delta}` : ""}`;
3027 }),
3028 "",
3029 ];
3030 const errors = groups(
3031 rows.filter((row) => row.error !== undefined),
3032 (row) => row.id,
3033 );
3034 const errorNote = `- Errored: ${errors.map(([id, failed]) => `${id} ×${failed.length}`).join(", ")}. These samples did not finish, even after retries, so pass rates leave them out; tokens they spent are counted.`;
3035 const nativeNote =
3036 "- native-subagent cases are approximate: the harness gives subagents their own system prompt, which this runner does not rebuild. T3's append is left out and its SubagentStart hook context added after the task.";
3037 const identityNote = `- The captured Claude Code identity line is replaced with "${NEUTRAL_IDENTITY}"`;
3038 const finished = rows.filter((row) => row.error === undefined).length;
3039 return [
3040 "# MCP tool-use eval\n",
3041 `- ${options.header}`,
3042 `- Pass: ${percent(passRate(rows))} of ${finished} samples.`,
3043 ...(errors.length > 0 ? [errorNote] : []),
3044 codex
3045 ? "- Cost: the agent is not priced. The Codex model provider (see README) may use a subscription or bill an API key per token; agent tokens are below."
3046 : `- Estimated cost $${cost.toFixed(2)}, $${(cost / rows.length).toFixed(3)} per sample.`,
3047 `- Agent tokens: ${agent.tokens}${codex ? "." : ` ($${agent.cost.toFixed(2)}).`}`,
3048 `- Shell simulator (${options.shellModel}) tokens: ${shell.tokens} (API-price estimate $${shell.cost.toFixed(2)}).`,
3049 ...(codex
3050 ? [
3051 "- Code-mode tools use case mocks and a simulated machine. Native subagent prompts and asynchronous notifications are approximate.",
3052 ]
3053 : [identityNote]),
3054 ...(rows.some((row) => row.role === "native-subagent") ? [nativeNote] : []),
3055 "",
3056 ...table("Cases", (row) => row.id, before !== undefined),
3057 ...table("By kind", (row) => row.kind),
3058 ...table("By pattern", (row) => row.pattern),
3059 ].join("\n");
3060 };
3061
3062 const PositiveInt = Schema.FiniteFromString.check(Schema.isInt(), Schema.isGreaterThan(0));
3063 const PositiveIntOrZero = Schema.FiniteFromString.check(
3064 Schema.isInt(),
3065 Schema.isGreaterThanOrEqualTo(0),
3066 );
3067 const Flags = Schema.Struct({
3068 model: Schema.Literals(Object.keys(MODELS) as Array<Model>),
3069 template: Schema.String,
3070 cases: Schema.optional(Schema.Array(Schema.String)),
3071 out: Schema.optional(Schema.String),
3072 baseline: Schema.optional(Schema.String),
3073 case: Schema.optional(Schema.Array(Schema.String)),
3074 samples: Schema.optional(PositiveInt),
3075 concurrency: PositiveInt,
3076 "max-steps": Schema.optional(PositiveInt),
3077 effort: Schema.optional(Schema.Literals(EFFORTS)),
3078 "shell-model": Schema.Literals([
3079 ...(Object.keys(PRICES) as Array<PricedModel>),
3080 ...PROXY_SHELL_MODELS,
3081 ]),
3082 "user-instructions": Schema.Literals(["keep", "drop"]),
3083 arm: Schema.Literals(ARMS),
3084 "dry-run": Schema.Boolean,
3085 "stop-on-regression": Schema.Boolean,
3086 "regression-budget": Schema.optional(PositiveIntOrZero),
3087 context: Schema.Boolean,
3088 });
3089 const decodeFlags = Schema.decodeUnknownEffect(Flags);
3090 const decodeCapture = Schema.decodeUnknownEffect(Schema.fromJsonString(Capture));
3091 const decodeCodexCapture = Schema.decodeUnknownEffect(Schema.fromJsonString(CodexCapture));
3092 const decodeCase = Schema.decodeUnknownEffect(Schema.fromJsonString(McpToolUseCase));
3093 const decodeBaselineRow = Schema.decodeUnknownEffect(
3094 Schema.fromJsonString(
3095 Schema.Struct({
3096 id: Schema.String,
3097 pass: Schema.Boolean,
3098 error: Schema.optional(Schema.String),
3099 }),
3100 ),
3101 );
3102 const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
3103 const string = { type: "string" } as const;
3104
3105 const main = Effect.gen(function* () {
3106 const { values } = NodeUtil.parseArgs({
3107 options: {
3108 model: { ...string, default: "claude-sonnet-5-5" },
3109 template: string,
3110 cases: { ...string, multiple: true },
3111 out: string,
3112 baseline: string,
3113 case: { ...string, multiple: true },
3114 // Unset: 2 samples per case, and a 3rd when those 2 disagree.
3115 samples: string,
3116 // Samples in flight at once, across all cases.
3117 concurrency: { ...string, default: "8" },
3118 // Unset: 6 for Claude, 10 for Codex, whose code-mode cells take more steps.
3119 "max-steps": string,
3120 effort: string,
3121 "shell-model": { ...string, default: "claude-haiku-5-5" },
3122 // T3's text must work for every user, so the captured CLAUDE.md is dropped by default.
3123 "user-instructions": { ...string, default: "drop" },
3124 // `raw` runs without T3's PR, CI and issue tools, to compare against the CLIs.
3125 arm: { ...string, default: "tools" },
3126 "dry-run": { type: "boolean", default: false },
3127 "stop-on-regression": { type: "boolean", default: false },
3128 "regression-budget": string,
3129 // Prints the size of T3's text and tool schemas, then exits. Needs no template or key.
3130 context: { type: "boolean", default: false },
3131 },
3132 });
3133 const flags = yield* decodeFlags({ template: "", ...values });
3134 if (flags.context) {
3135 process.stdout.write(`${t3ContextReport(flags.model)}\n`);
3136 return;
3137 }
3138 if (flags["stop-on-regression"] && flags.baseline === undefined) {
3139 return yield* Effect.die(new Error("--stop-on-regression needs --baseline."));
3140 }
3141 const fs = yield* FileSystem.FileSystem;
3142 const path = yield* Path.Path;
3143 const readNdjson = <A, E>(file: string, decode: (line: string) => Effect.Effect<A, E>) =>
3144 fs.readFileString(file).pipe(
3145 Effect.map((content) => content.split("\n").filter((line) => line.trim() !== "")),
3146 Effect.flatMap((lines) => Effect.forEach(lines, (line) => decode(line))),
3147 );
3148 const casesDir = path.join(import.meta.dirname, "cases");
3149 const caseFiles =
3150 flags.cases ??
3151 (yield* fs.readDirectory(casesDir))
3152 .filter((name) => name.endsWith(".ndjson"))
3153 .toSorted()
3154 .map((name) => path.join(casesDir, name));
3155 const cases = (yield* Effect.forEach(caseFiles, (file) => readNdjson(file, decodeCase))).flat();
3156 const wanted = flags.case?.flatMap((ids) => ids.split(","));
3157 const codex = isCodex(flags.model);
3158 const harness = codex ? "codex" : "claude";
3159 const selectedCases = cases.filter(
3160 ({ id, setup }) =>
3161 (wanted === undefined || wanted.includes(id)) &&
3162 (setup.harness === undefined || setup.harness === harness),
3163 );
3164 if (selectedCases.length === 0) return yield* Effect.die(new Error("No case matches --case."));
3165 // The raw arm has no PR or issue tools, so a case whose prior turns call one cannot be replayed.
3166 const replaysForgeTool = (testCase: McpToolUseCase) =>
3167 (testCase.setup.priorTurns ?? []).some((turn) =>
3168 (turn.toolCalls ?? []).some((call) => forgeToolNames.has(`${T3_PREFIX}${call.tool}`)),
3169 );
3170 const unreplayable =
3171 flags.arm === "raw" ? selectedCases.filter(replaysForgeTool).map(({ id }) => id) : [];
3172 if (unreplayable.length > 0) {
3173 yield* Effect.logWarning(
3174 `--arm raw skips ${unreplayable.length} cases whose prior turns call PR or issue tools: ${unreplayable.join(", ")}`,
3175 );
3176 }
3177 const armCases = selectedCases.filter(({ id }) => !unreplayable.includes(id));
3178 if (armCases.length === 0)
3179 return yield* Effect.die(new Error("No selected case can run with --arm raw."));
3180 const selected = yield* Effect.forEach(armCases, (testCase) =>
3181 Effect.gen(function* () {
3182 const priorTurns = yield* Effect.forEach(testCase.setup.priorTurns ?? [], (turn) =>
3183 Effect.gen(function* () {
3184 const toolCalls = yield* Effect.forEach(turn.toolCalls ?? [], (call) =>
3185 call.resultFile === undefined
3186 ? Effect.succeed(call)
3187 : fs
3188 .readFileString(path.resolve(import.meta.dirname, "../..", call.resultFile))
3189 .pipe(Effect.map((result) => ({ ...call, result }))),
3190 );
3191 return { ...turn, toolCalls };
3192 }),
3193 );
3194 return {
3195 ...testCase,
3196 expect: flags.arm === "raw" ? (testCase.expectRaw ?? testCase.expect) : testCase.expect,
3197 setup: {
3198 ...testCase.setup,
3199 priorTurns,
3200 },
3201 };
3202 }),
3203 );
3204 // Trimmed captures without user data ship beside the cases. `--template` also takes a raw
3205 // CLIProxyAPI capture.
3206 const templatePath =
3207 flags.template ||
3208 path.join(import.meta.dirname, "templates", codex ? "codex.json.gz" : "claude.json.gz");
3209 const templateBytes = yield* fs.readFile(templatePath);
3210 const template = new TextDecoder().decode(
3211 templatePath.endsWith(".gz") ? NodeZlib.gunzipSync(templateBytes) : templateBytes,
3212 );
3213 const keepUserInstructions = flags["user-instructions"] === "keep";
3214 const buildFor = (
3215 role: McpToolUseCase["setup"]["role"],
3216 deviceAccess: boolean,
3217 browserAccess: boolean,
3218 links: boolean,
3219 forge: boolean,
3220 ): Effect.Effect<Base, Schema.SchemaError> =>
3221 codex
3222 ? decodeCodexCapture(template).pipe(
3223 Effect.map((capture) =>
3224 buildCodexBase(capture.request.provider_native, {
3225 model: flags.model,
3226 keepUserInstructions,
3227 effort: flags.effort,
3228 subagent: role === "delegated-child",
3229 links,
3230 forge,
3231 deviceAccess,
3232 browserAccess,
3233 arm: flags.arm,
3234 }),
3235 ),
3236 )
3237 : decodeCapture(template).pipe(
3238 Effect.flatMap((capture) => {
3239 const built = buildBase(capture.request.provider_native, {
3240 model: flags.model,
3241 keepUserInstructions,
3242 subagent: role === "delegated-child",
3243 nativeSubagent: role === "native-subagent",
3244 links,
3245 forge,
3246 deviceAccess,
3247 browserAccess,
3248 arm: flags.arm,
3249 });
3250 return role === "native-subagent"
3251 ? Effect.promise(() =>
3252 t3SubagentContext(flags.model, deviceAccess, browserAccess),
3253 ).pipe(Effect.map((subagentContext): Base => ({ ...built, subagentContext })))
3254 : Effect.succeed(built);
3255 }),
3256 );
3257 // Codex has no native subagents; its cases with that role run as top-level threads.
3258 const baseKey = ({ setup }: McpToolUseCase) =>
3259 `${codex && setup.role === "native-subagent" ? "top" : setup.role}:${setup.world?.deviceAgentAccess !== false}:${setup.world?.browserAgentAccess !== false}:${setup.linkedEnvironments === true}:${setup.forge !== false}`;
3260 const base = yield* buildFor("top", true, true, false, true);
3261 const bases = new Map<string, Base>([["top:true:true:false:true", base]]);
3262 for (const testCase of selected) {
3263 const key = baseKey(testCase);
3264 if (bases.has(key)) continue;
3265 const [role = "top", device, browser, links, forge] = key.split(":") as [
3266 McpToolUseCase["setup"]["role"],
3267 string,
3268 string,
3269 string,
3270 string,
3271 ];
3272 bases.set(
3273 key,
3274 yield* buildFor(
3275 role,
3276 device === "true",
3277 browser === "true",
3278 links === "true",
3279 forge === "true",
3280 ),
3281 );
3282 }
3283 const baseFor = (testCase: McpToolUseCase) => bases.get(baseKey(testCase)) ?? base;
3284 const effort = flags.effort ?? base.codex?.reasoning.effort ?? "high";
3285 if (flags["dry-run"]) {
3286 // One JSON line per case. Without --case this also checks that every case builds.
3287 for (const testCase of selected) {
3288 const sample = makeSample(testCase, baseFor(testCase));
3289 yield* Effect.scoped(Effect.andThen(withDevices(sample), withBackgroundWait(sample)));
3290 const messages = buildMessages(sample);
3291 const body = sample.base.codex
3292 ? codexRequest(sample.base, codexMessages(messages))
3293 : requestBody(sample, effort, messages);
3294 process.stdout.write(`${yield* encodeJson(body)}\n`);
3295 }
3296 return;
3297 }
3298 const apiKey = yield* Config.String("ANTHROPIC_API_KEY").pipe(Config.withDefault(""));
3299 const out = flags.out;
3300 if ((!apiKey && !codex) || out === undefined) {
3301 const message =
3302 "Set ANTHROPIC_API_KEY and pass --out <directory>. ANTHROPIC_AUTH_TOKEN and ANTHROPIC_BASE_URL are never used.";
3303 return yield* Effect.die(new Error(message));
3304 }
3305 const config: Config = {
3306 apiKey,
3307 model: flags.model,
3308 ...(codex || !isPricedModel(flags["shell-model"]) ? { proxy: proxyConfig() } : {}),
3309 effort,
3310 maxSteps: flags["max-steps"] ?? (codex ? 10 : 6),
3311 shellModel: flags["shell-model"],
3312 };
3313 const previous =
3314 flags.baseline === undefined ? [] : yield* readNdjson(flags.baseline, decodeBaselineRow);
3315 const baseline = new Map(
3316 groups(previous, (row) => row.id)
3317 .map(([id, r]) => [id, passRate(r)] as const)
3318 .filter(([, rate]) => !Number.isNaN(rate)),
3319 );
3320 // Only a case that passed every baseline sample can stop a run; flaky ones are just reported.
3321 const passedBefore = (id: string) => baseline.get(id) === 1;
3322 // Before the paid runs, so a missing git cannot lose their results.
3323 const commit = yield* Effect.sync(() =>
3324 Option.getOrElse(
3325 Option.liftThrowable(() =>
3326 NodeChildProcess.execFileSync("git", ["describe", "--always", "--dirty"], {
3327 encoding: "utf8",
3328 }).trim(),
3329 )(),
3330 () => "unknown",
3331 ),
3332 );
3333 const inFlight = yield* Semaphore.make(flags.concurrency);
3334 const runOne = (testCase: McpToolUseCase) => {
3335 const { id, kind, pattern } = testCase;
3336 const identity = {
3337 id,
3338 kind,
3339 pattern,
3340 role: testCase.setup.role,
3341 arm: flags.arm,
3342 shellModel: config.shellModel,
3343 };
3344 const usage = { usage: emptyUsage(), shellUsage: emptyUsage() };
3345 const privateValues = [config.apiKey, ...Object.values(config.proxy?.headers ?? {})]
3346 .flatMap((value) => [value, value.replace(/^Bearer /i, "")])
3347 .filter(Boolean);
3348 return runSample(testCase, config, baseFor(testCase), usage).pipe(
3349 inFlight.withPermits(1),
3350 Effect.matchCause({
3351 // Not graded: a sample that errors is left out of pass rates, with its tokens so far.
3352 onFailure: (cause) => ({
3353 ...identity,
3354 ...usage,
3355 pass: false,
3356 error: privateValues.reduce(
3357 (message, value) => message.replaceAll(value, "[redacted]"),
3358 Cause.pretty(cause),
3359 ),
3360 }),
3361 onSuccess: (sample) => {
3362 const checks = gradeSample(
3363 testCase.expect,
3364 sample,
3365 pastedText(testCase),
3366 worldFilePaths(testCase),
3367 );
3368 const { secret } = testCase.expect;
3369 return {
3370 ...identity,
3371 pass: checks.every((check) => check.pass),
3372 checks,
3373 // Not graded: how the agent handled a secret it needed, for a person reading the run.
3374 ...(secret ? { secret: secretDiagnostics(sample, secret) } : {}),
3375 ...sample,
3376 };
3377 },
3378 }),
3379 Effect.tap((row) =>
3380 Effect.log(`${row.id}: ${"error" in row ? "ERROR" : row.pass ? "pass" : "FAIL"}`),
3381 ),
3382 );
3383 };
3384 // Without --samples, a case whose 2 samples disagree gets a 3rd to settle it.
3385 const runCase = Effect.fn("evaluateMcpToolUse.runCase")(function* (testCase: McpToolUseCase) {
3386 const samples = Array.from({ length: flags.samples ?? 2 }, () => testCase);
3387 const first = yield* Effect.forEach(samples, runOne, { concurrency: "unbounded" });
3388 const split = flags.samples === undefined && new Set(first.map((row) => row.pass)).size > 1;
3389 return split ? [...first, yield* runOne(testCase)] : first;
3390 });
3391 // --stop-on-regression runs cases that passed in the baseline first and stops once more than
3392 // --regression-budget of them fail, so a copy change that breaks passing cases fails fast.
3393 // A budget lets a quirky model miss a case or two without ending the run.
3394 const stopEarly = flags["stop-on-regression"];
3395 const budget = flags["regression-budget"] ?? 0;
3396 const ordered = stopEarly
3397 ? selected.toSorted((a, b) => Number(passedBefore(b.id)) - Number(passedBefore(a.id)))
3398 : selected;
3399 const rows: Array<Effect.Success<ReturnType<typeof runOne>>> = [];
3400 const regressions: Array<string> = [];
3401 yield* Effect.forEach(
3402 ordered,
3403 (testCase) =>
3404 runCase(testCase).pipe(
3405 Effect.flatMap((caseRows) => {
3406 rows.push(...caseRows);
3407 // A case whose samples all errored has no rate, so it is no regression either.
3408 if (!stopEarly || !passedBefore(testCase.id) || !(passRate(caseRows) <= 0.5)) {
3409 return Effect.void;
3410 }
3411 regressions.push(testCase.id);
3412 return regressions.length > budget ? Effect.fail(testCase.id) : Effect.void;
3413 }),
3414 ),
3415 { concurrency: flags.concurrency, discard: true },
3416 ).pipe(Effect.ignore);
3417 const samples = flags.samples === undefined ? "2-3 (adaptive)" : flags.samples;
3418 const header = `Commit \`${commit}\`, ${flags.model} at ${effort} effort, user instructions ${flags["user-instructions"]}, arm ${flags.arm}, ${samples} samples per case, template \`${path.basename(templatePath)}\``;
3419 const finished = new Set(rows.map((row) => row.id)).size;
3420 const stopped =
3421 regressions.length === 0
3422 ? []
3423 : [
3424 `- Regressions (passed every baseline sample, now fail): ${regressions.map((id) => `\`${id}\``).join(", ")}.${regressions.length > budget ? ` Stopped with ${finished} of ${selected.length} cases finished.` : ""}`,
3425 ];
3426 const summary = summarize(
3427 rows,
3428 {
3429 header: [header, t3ContextReport(flags.model), ...stopped].join("\n"),
3430 shellModel: config.shellModel,
3431 model: config.model,
3432 },
3433 flags.baseline === undefined ? undefined : baseline,
3434 );
3435 yield* fs.makeDirectory(out, { recursive: true });
3436 const lines = yield* Effect.forEach(rows, (row) => encodeJson(row));
3437 yield* fs.writeFileString(path.join(out, "results.jsonl"), `${lines.join("\n")}\n`);
3438 yield* fs.writeFileString(path.join(out, "summary.md"), summary);
3439 yield* Effect.log(`Wrote ${rows.length} samples to ${out}.\n\n${summary}`);
3440 });
3441
3442 if (import.meta.main) {
3443 main.pipe(
3444 Effect.provide(
3445 Layer.mergeAll(NodeServices.layer, PlatformNodeCrypto.layer, FetchHttpClient.layer),
3446 ),
3447 NodeRuntime.runMain,
3448 );
3449 }