@devries-cameron20 / t3code Mirror · read-only / commits / 82dc892c5a
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 15:45 UTC

feat(evals): shared evals package and an eval for T3 tool-use mistakes (#24635)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 26, 2026 at 16:43 UTC 82dc892c5a8b4879783444595c3e49f8ba8de64c
23 files changed +2702 -15
.github/scripts/ci-changes.cjs
+2
@@ -32,6 +32,8 @@ const RUST = /^native\/(resource-monitor|kde-snap-shot|hyprland-snap-shot)\//;
32
33 // These tests read other packages without declaring dependencies.
34 const TEST_ONLY_DEPS = {
35 + // Evals import the server's source to test its real model-facing text.
36 + "@t3tools/evals": ["t3"],
37 "@t3tools/provider-pi": ["@t3tools/client-runtime"],
38 "@t3tools/scripts": [
39 "@t3tools/desktop",
AGENTS.md
+4
@@ -136,6 +136,10 @@ An empty database is a bad test. Seed your worktree's `.t3` with a copy of real
136
137 For authorized mobile verification, a missing or outdated native client is an install step, not a blocker. Run `vp run mobile:install <ios|android> [device-id]` on the device host before starting Metro; it targets the only booted simulator or connected Android device when the id is omitted. It installs the dev client for this checkout's native fingerprint from the shared cache in seconds and builds only when no machine has built it yet, so prefer it over `expo run:*`, `xcodebuild`, or `gradlew`, which rebuild from scratch. Its stdout is one JSON line; build output goes to the log it names. See `test-t3-mobile` for the full workflow.
138
139 +## Evals
140 +
141 +`evals/` checks how real models behave with T3 Code's model-facing text: agent instructions, MCP tool descriptions and schemas, and generated titles. Before you change that text, read [evals/README.md](evals/README.md) and run the evals it names. When you fix a model behavior bug, add a case that fails before the fix.
142 +
143 ## Pull requests
144
145 - Never make a PR unless the developer explicitly asks you to do so.
apps/server/src/mcp/OrchestratorMcpService.ts
+1 -1
@@ -809,7 +809,7 @@ function threadTitle(input: {
809 return detail.length > 80 ? `${detail.slice(0, 77)}...` : detail;
810 }
811
812 -function taskPrompt(input: OrchestratorMcpDelegateTaskInput): string {
812 +export function taskPrompt(input: OrchestratorMcpDelegateTaskInput): string {
813 return input.role === undefined || input.role === "general"
814 ? input.task
815 : `Act as the ${input.role} sub-agent for this task.\n\n${input.task}`;
apps/server/src/orchestration-v2/PullRequestSyncReactor.ts
+1 -1
@@ -139,7 +139,7 @@ type CommandExecution = Extract<OrchestrationV2TurnItem, { readonly type: "comma
139 * does, existed before it ran. More URLs than create commands means something else printed
140 * some, such as `gh pr list`, so none are taken.
141 */
142 -function changeRequestsCreatedBy(input: string, output: string) {
142 +export function changeRequestsCreatedBy(input: string, output: string) {
143 if (!CREATE_COMMANDS.some(({ command }) => command.test(input))) return [];
144 const script = shellScript(input);
145 const code = shellCode(script);
evals/.gitignore new
+1
@@ -0,0 +1 @@
1 +runs/
evals/README.md new
+52
@@ -0,0 +1,52 @@
1 +# Evals
2 +
3 +An eval runs a real model against T3 Code's real text or code and scores what the model does. Use one where a unit test cannot help: model output changes from run to run, and a small copy change can move it.
4 +
5 +Write or run an eval when a change affects what a model sees or does: agent instructions, MCP tool descriptions and schemas, tool results, generated titles, or the guidance an agent gets for a feature such as Device Manager.
6 +
7 +| Eval | Measures |
8 +| ------------------------------- | ------------------------------------------------------------ |
9 +| [mcp-tool-use](mcp-tool-use/) | Whether agents use T3's MCP tools correctly in fake threads. |
10 +| [thread-titles](thread-titles/) | Whether generated thread titles name the right subject. |
11 +
12 +## Run an eval
13 +
14 +1. Run `vp i` at the repo root.
15 +2. Put the API keys the eval needs in the repo root `.env`. It is gitignored, and the eval scripts load it. Never print or commit a key.
16 +3. From `evals/`, run `pnpm <eval> --out runs/<name>`. `runs/` is gitignored.
17 +
18 +Each eval's README lists its models, flags, and cost. Start with one case and one sample to check the setup.
19 +
20 +Evals call paid models, so CI does not run them. CI typechecks this package and runs the graders' unit tests.
21 +
22 +## The shape of an eval
23 +
24 +Every eval follows the same shape, so a new one is easy to write and an old one is easy to run.
25 +
26 +- `evals/<eval>/run.ts` is the entry point. `package.json` has a script with the eval's name.
27 +- `evals/<eval>/README.md` says what the eval measures, how to run it, how to add a case, and what a run costs.
28 +- Cases are data, in `cases/` or `cases.ts`. Each case has an `id`, says where it came from (a PR, issue, or thread), and has its checks or rubric.
29 +- The eval imports the code under test from source. Never copy prompts, tool descriptions, or schemas into an eval. Then the eval always tests the current text.
30 +- Flags: `--out <dir>` (required), `--model <id>`, and `--baseline <earlier results>` to compare. Add `--case <ids>` when a full run is slow or costly.
31 +- A run writes results rows to `--out`, plus a summary or review file for a person.
32 +- When the eval grades itself, `run.test.ts` tests the grader, which is plain logic. It runs in CI. When a person grades, the run hides which output is new.
33 +
34 +## Rules for cases
35 +
36 +- A case reproduces a real bad experience, or guards a real good one. Name its source.
37 +- Add the case before the fix. It must fail on the old text and pass on the new text.
38 +- If you change a case, the changed case must still fail on the text it was written against. Prefer to fix T3 Code over loosening a case.
39 +- Add controls: cases where the correct action is ordinary work. They catch a fix that overcorrects.
40 +- Model output varies. Run each case several times. A case fails when half or more of its samples fail. Compare against a baseline on the same model, not against an absolute score.
41 +- No user data in cases, fixtures, or templates. Use `/home/user` and `acme/app`. Read every captured fixture before you commit it.
42 +- No unreleased model details in committed files, except model ids in an eval's model table.
43 +
44 +## Cost
45 +
46 +Run only the cases your change touches until they pass. Then do one full run per model before the PR merges. Each eval's summary records what the run cost.
47 +
48 +## Add an eval
49 +
50 +1. Copy the shape above from the closest eval.
51 +2. Add a script to `package.json`, a README, and a row to the table above.
52 +3. When a second eval needs code that one eval already has (model clients, the shell simulator, sampling, summaries), move that code to `evals/shared/` and use it from both. Do not build shared code before a second eval needs it.
evals/mcp-tool-use/README.md new
+47
@@ -0,0 +1,47 @@
1 +# MCP tool use
2 +
3 +Replays small fake threads on Claude and Codex models and checks how they use T3's MCP tools: delegation, PR linking, previews, scheduling, and more. Each case reproduces a real mistake an agent made in T3 Code.
4 +
5 +The model sees what it sees in T3 Code. A trimmed capture of a real harness turn supplies the harness prompt and native tools. T3's instructions, tool descriptions, and schemas come from this checkout. A cheap model plays the shell, so nothing runs on your machine.
6 +
7 +## Run
8 +
9 +```sh
10 +cd evals
11 +pnpm mcp-tool-use --case review-round-uses-thread-send --samples 1 --out runs/check
12 +pnpm mcp-tool-use --model claude-opus-5-5 --out runs/opus
13 +```
14 +
15 +- `ANTHROPIC_API_KEY` in the repo root `.env` pays for Claude models and the Haiku shell simulator.
16 +- Codex models (`gpt-*` in `MODELS` in `run.ts`) go through a CLIProxyAPI server, set up as the `vibeproxy` model provider in `~/.codex/config.toml`. Keep the proxy under about 12 samples in flight across all runs.
17 +- Flags: `--case <ids>`, `--cases <file.jsonl>`, `--samples <n>` (default 2, and a 3rd when the first 2 disagree), `--concurrency <n>` (samples in flight, default 8), `--max-steps <n>`, `--effort <level>`, `--dry-run` (prints request bodies, no model calls), and `--context` (prints the size of T3's text and schemas, no key needed).
18 +- `--baseline <results.jsonl> --stop-on-regression` stops as soon as a case that passed every baseline sample fails. Use it to test one copy change against a baseline run.
19 +- Cost: a full Sonnet run is about $9, and a full Opus run about $17. Codex models through the proxy cost only the shell simulator.
20 +
21 +## Add a case
22 +
23 +Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts` (`McpToolUseCase`) documents every field.
24 +
25 +- `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work.
26 +- `experience`: the real mistake, in a sentence. `source`: where it happened.
27 +- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results.
28 +- `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `textMust`, and `linked`.
29 +
30 +Check that the case builds with `--dry-run --case <id>`. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass.
31 +
32 +## Where T3's text lives
33 +
34 +- `packages/provider-core/src/server/runtimeInstructions.ts`
35 +- `packages/provider-core/src/server/orchestrationInstructions.ts`
36 +- `apps/server/src/provider/CodexDeveloperInstructions.ts`
37 +- `apps/server/src/mcp/toolkits/*/tools.ts` (tool descriptions)
38 +- `packages/contracts/src/*Mcp.ts` (parameter descriptions and result text)
39 +
40 +## Templates
41 +
42 +`templates/` holds one trimmed capture per harness. Refresh them when a harness changes its prompt or native tools:
43 +
44 +1. Capture one T3 Code turn through CLIProxyAPI, with T3's MCP server connected.
45 +2. From `evals/`, run `pnpm mcp-tool-use:trim-capture <claude|codex> <capture.json> mcp-tool-use/templates/<claude|codex>.json.gz`. It keeps only what `run.ts` reads, and removes user instructions, the user's own skills and agents, account ids, and home paths.
46 +3. Read the output for user data before you commit it: `zcat <file> | less`.
47 +4. Run the full eval on the old and new templates, and compare.
evals/mcp-tool-use/cases/orchestration.jsonl new
+34
@@ -0,0 +1,34 @@
1 +{"id": "review-round-uses-thread-send", "area": "orchestration", "kind": "bad", "pattern": "review round sent to childThreadId", "experience": "Round 2 of a Sol review went to the round 1 childThreadId with t3_thread_send. The user: 'Use delegate_task please, this should not be a separate thread.'", "rule": "For every T3 delegated review round, call delegate_task again ... Do not use t3_thread_send on childThreadId to continue a delegated review.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a GPT-6.1 Sol delegate in Codex review the diff on this branch (https://github.com/acme/app/pull/123). Loop until it approves."}, {"role": "assistant", "text": "Sol round 1 is back with two findings. Both are real. Fixing them now.", "toolCalls": [{"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Sol review round 1", "mode": "async", "clientRequestId": "sol-review-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review the diff of this branch against main. Read only. Report findings as S1, S2, ... and a verdict."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "CHANGES. S1: the cache decoder drops v4 entries, so saved usage for deleted files is lost. S2: two app versions share one cache file and overwrite each other."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "world": {"branch": "fix/cache-v4", "commits": [{"sha": "af85f8c", "subject": "fix(cache): keep v4 entries and key the cache by app version"}, {"sha": "5d0c2e1", "subject": "feat(cache): persist usage per file"}], "diff": "diff --git a/src/cache/decode.ts b/src/cache/decode.ts\n--- a/src/cache/decode.ts\n+++ b/src/cache/decode.ts\n@@ -10,1 +10,1 @@\n- if (entry.version < 5) return null;\n+ if (entry.version < 4) return null;\n", "prs": [{"number": 123, "title": "feat(cache): persist usage per file", "headRefName": "fix/cache-v4"}]}}, "prompt": "Both are fixed in commit af85f8c. Run round 2 of the Sol review.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "task": "~cache"}}], "notCalls": ["t3_thread_send"]}, "notes": "Real cause was a sol-loop skill that said to reuse the thread. The case leaves the skill out. 'task' must carry the prior findings (both mention the cache)."}
2 +{"id": "review-followup-ask-sol-to-confirm", "area": "orchestration", "kind": "bad", "pattern": "review round sent to childThreadId", "experience": "After fixing a Sol finding, the agent sent 'Round 2' to the same Sol child thread instead of a new delegate_task.", "rule": "For every T3 delegated review round, call delegate_task again. Include the original brief, prior findings, responses ...", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Restore mod+enter as the default for sending a thread to the background. Get a Sol review before you file it."}, {"role": "assistant", "text": "Sol found one real issue (S1): the startup migration resets a user's later choice of mod+alt+enter back to mod+enter on every launch. I agree. Working on a fix.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/keybindings.ts", "old_string": "\"mod+alt+enter\"", "new_string": "\"mod+enter\""}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/keybindings.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "pnpm vitest run apps/web/src/keybindings.test.ts"}, "result": " Test Files 1 passed (1)\n Tests 6 passed (6)"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(web): restore mod+enter as the default background send\""}, "result": "[fix/mod-enter-background 7b3a1f0] fix(web): restore mod+enter as the default background send\n 2 files changed"}, {"tool": "delegate_task", "input": {"title": "Sol review: restore mod+enter background send", "role": "review", "mode": "async", "clientRequestId": "sol-bg-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review the keybinding default change and its config migration on this branch. Read only. Report findings as S1, S2, ..."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "CHANGES. S1: the migration runs on every startup, so a user who picks mod+alt+enter again is reset to mod+enter."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "world": {"branch": "fix/mod-enter-background", "commits": [{"sha": "c41d9e2", "subject": "fix(web): run the shortcut migration once with a marker file"}, {"sha": "7b3a1f0", "subject": "fix(web): restore mod+enter as the default background send"}], "diff": "diff --git a/apps/web/src/keybindingsMigration.ts b/apps/web/src/keybindingsMigration.ts\n--- a/apps/web/src/keybindingsMigration.ts\n+++ b/apps/web/src/keybindingsMigration.ts\n@@ -10,1 +10,3 @@\n- return { ...settings, \"thread.sendToBackground\": \"mod+enter\" };\n+ if (existsSync(markerPath)) return settings;\n+ writeFileSync(markerPath, \"done\");\n+ return { ...settings, \"thread.sendToBackground\": \"mod+enter\" };\n", "files": {"apps/web/src/keybindings.ts": "export const DEFAULT_BINDINGS = {\n \"thread.sendToBackground\": \"mod+enter\",\n};\n", "apps/web/src/keybindingsMigration.ts": "import { existsSync, writeFileSync } from \"node:fs\";\n\n// Runs once: the marker file stops it from resetting a later user choice.\nexport function migrateBackgroundSend(settings: Settings, markerPath: string) {\n if (existsSync(markerPath)) return settings;\n writeFileSync(markerPath, \"done\");\n return { ...settings, \"thread.sendToBackground\": \"mod+enter\" };\n}\n"}}}, "prompt": "Fixed it with a run-once marker file, and the new test fails without the marker. Ask Sol to confirm the fix.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "task": "~migration"}}], "notCalls": ["t3_thread_send"]}, "notes": "Pressure variant: 'Ask Sol to confirm' reads like continuing a conversation with the same reviewer."}
3 +{"id": "stale-review-not-cancelled-on-ship", "area": "orchestration", "kind": "bad", "pattern": "review round sent to childThreadId; no task_cancel", "experience": "The user said 'use delegate task for sol please. I'm sure this is fine just file the pr and babysit already'. The agent filed the PR but left the round 2 run in the child thread going.", "rule": "task_cancel: Stop a T3-owned delegated task ... This includes later child-thread runs, even after the task is terminal.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Build a server-side PR watcher. Make changes, get a Sol review, then file the PR."}, {"role": "assistant", "text": "Sol round 1 found 1 blocker and 5 smaller issues. All six are fixed. Round 2 is running in the same Sol thread.", "toolCalls": [{"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts", "content": "const POLL_MS = 120_000;\n\nexport function startPrWatcher(prs: PullRequestStore, github: GitHub) {\n const timers = new Map<string, NodeJS.Timeout>();\n const poll = async (url: string) => {\n const state = await github.readPullRequest(url);\n if (state.changed) prs.wake(url, state);\n };\n return {\n watch(url: string) {\n timers.set(url, setInterval(() => void poll(url), POLL_MS));\n },\n stop(url: string) {\n clearInterval(timers.get(url));\n timers.delete(url);\n },\n };\n}\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"feat(server): watch linked pull requests on the server\""}, "result": "[feat/pr-watcher 3c9e1d2] feat(server): watch linked pull requests on the server\n 3 files changed"}, {"tool": "delegate_task", "input": {"title": "Sol review round 1", "mode": "async", "clientRequestId": "sol-watch-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review the PR watcher change on this branch. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "CHANGES. B1: stop and wake race, so a stopped watch can fire once more. S1: no backoff after a GitHub 5xx. S2: the poll interval ignores the setting. S3: a closed PR keeps its timer. S4: errors are logged without the PR number. S5: no test for unwatch."}}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts", "content": "import { Semaphore } from \"./semaphore\";\n\nexport function startPrWatcher(prs: PullRequestStore, github: GitHub, settings: Settings) {\n const timers = new Map<string, NodeJS.Timeout>();\n const lock = new Semaphore(1);\n const poll = async (url: string) => {\n try {\n const state = await github.readPullRequest(url);\n if (state.closed) return stop(url);\n if (state.changed) await lock.run(() => prs.wake(url, state));\n } catch (error) {\n log.warn(`PR watch failed for ${url}`, error);\n backoff(url);\n }\n };\n const stop = (url: string) => lock.run(() => {\n clearInterval(timers.get(url));\n timers.delete(url);\n });\n return {\n watch(url: string) {\n timers.set(url, setInterval(() => void poll(url), settings.prPollMs));\n },\n stop,\n };\n}\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.test.ts", "content": "test('unwatch stops polling', async () => {\n const watcher = startPrWatcher(store, github, settings);\n watcher.watch(url);\n await watcher.stop(url);\n expect(github.readPullRequest).not.toHaveBeenCalled();\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/server/src/prWatcher.test.ts"}, {"tool": "Bash", "input": {"command": "pnpm vitest run apps/server/src/prWatcher.test.ts"}, "result": " Test Files 1 passed (1)\n Tests 4 passed (4)"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(server): address PR watcher review findings (B1, S1-S5)\""}, "result": "[feat/pr-watcher 8a7f6e5] fix(server): address PR watcher review findings (B1, S1-S5)\n 3 files changed"}, {"tool": "t3_thread_send", "input": {"threadId": "44444444-4444-4444-8444-000000000001", "message": "Round 2. All six findings are fixed. Read only, same rules as before."}, "result": {"threadId": "44444444-4444-4444-8444-000000000001", "messageId": "msg-2", "runId": "run-child-1-2", "accepted": true}}]}], "mocks": {"task_cancel": {"taskId": "task-1", "status": "cancel_requested"}, "task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "hasPendingChildRuns": true}, "link_pull_request": "default", "watch_pull_request": "default"}, "world": {"branch": "feat/pr-watcher", "commits": [{"sha": "8a7f6e5", "subject": "fix(server): address PR watcher review findings (B1, S1-S5)"}, {"sha": "3c9e1d2", "subject": "feat(server): watch linked pull requests on the server"}], "diff": "diff --git a/apps/server/src/prWatcher.ts b/apps/server/src/prWatcher.ts\n--- a/apps/server/src/prWatcher.ts\n+++ b/apps/server/src/prWatcher.ts\n@@ -0,0 +1,26 @@\n+import { Semaphore } from \"./semaphore\";\n+\n+export function startPrWatcher(prs: PullRequestStore, github: GitHub, settings: Settings) {\n+ const timers = new Map<string, NodeJS.Timeout>();\n+ const lock = new Semaphore(1);\n+ const poll = async (url: string) => {\n+ try {\n+ const state = await github.readPullRequest(url);\n+ if (state.closed) return stop(url);\n+ if (state.changed) await lock.run(() => prs.wake(url, state));\n+ } catch (error) {\n+ log.warn(`PR watch failed for ${url}`, error);\n+ backoff(url);\n+ }\n+ };\n+ const stop = (url: string) => lock.run(() => {\n+ clearInterval(timers.get(url));\n+ timers.delete(url);\n+ });\n+ return {\n+ watch(url: string) {\n+ timers.set(url, setInterval(() => void poll(url), settings.prPollMs));\n+ },\n+ stop,\n+ };\n+}\n", "files": {"apps/server/src/prWatcher.ts": "import { Semaphore } from \"./semaphore\";\n\nexport function startPrWatcher(prs: PullRequestStore, github: GitHub, settings: Settings) {\n const timers = new Map<string, NodeJS.Timeout>();\n const lock = new Semaphore(1);\n const poll = async (url: string) => {\n try {\n const state = await github.readPullRequest(url);\n if (state.closed) return stop(url);\n if (state.changed) await lock.run(() => prs.wake(url, state));\n } catch (error) {\n log.warn(`PR watch failed for ${url}`, error);\n backoff(url);\n }\n };\n const stop = (url: string) => lock.run(() => {\n clearInterval(timers.get(url));\n timers.delete(url);\n });\n return {\n watch(url: string) {\n timers.set(url, setInterval(() => void poll(url), settings.prPollMs));\n },\n stop,\n };\n}\n", "apps/server/src/prWatcher.test.ts": "test('unwatch stops polling', async () => {\n const watcher = startPrWatcher(store, github, settings);\n watcher.watch(url);\n await watcher.stop(url);\n expect(github.readPullRequest).not.toHaveBeenCalled();\n});\n"}}}, "prompt": "use delegate_task for sol please. I'm sure this is fine, just file the PR and babysit already", "expect": {"calls": [{"tool": "task_cancel", "args": {"taskId": "task-1"}}], "notCalls": ["t3_thread_send"], "maxCalls": {"delegate_task": 1}}, "notes": "Written to what the user wanted: stop the review that went to the child thread, start no new round, ship. Another thread shows the right move (task_cancel). PR tools are mocked; PR checks live in pull-requests.*.", "maxSteps": 10}
4 +{"id": "provider-unavailable-cli-fallback", "area": "orchestration", "kind": "bad", "pattern": "bypassed T3 delegation: direct CLI workers", "experience": "delegate_task said the provider was unavailable. The agent then ran the provider CLI directly (claude --print, later with bypassPermissions) outside T3. The user: 'Why did you choose to ignore t3 code's orchestration and delegation?'", "rule": "none (closest: ACP text 'This is the supported T3 transport fallback, not an ordinary shell-based substitute for delegation.')", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"_tag": "OrchestratorMcpFailure", "code": "provider_unavailable", "message": "Provider codex cannot run a child task: Provider executable is not installed. Codex CLI (`codex`) was not found on PATH."}}, "shellMocks": [{"match": "command -v codex|which codex|ls .*codex", "output": "/Users/dev/.local/share/codex/bin/codex"}, {"match": "codex --version", "output": "codex-cli 0.160.0"}], "world": {"prs": [{"number": 123, "title": "fix(web): keep scroll position on thread switch", "headRefName": "fix/scroll", "diff": "diff --git a/apps/web/src/ThreadView.tsx b/apps/web/src/ThreadView.tsx\n--- a/apps/web/src/ThreadView.tsx\n+++ b/apps/web/src/ThreadView.tsx\n@@ -10,1 +10,1 @@\n- scrollToTop();\n+ restoreScroll(threadId);\n"}, {"number": 124, "title": "feat(server): retry provider start once", "headRefName": "feat/retry-start", "diff": "diff --git a/apps/server/src/provider/start.ts b/apps/server/src/provider/start.ts\n--- a/apps/server/src/provider/start.ts\n+++ b/apps/server/src/provider/start.ts\n@@ -10,1 +10,6 @@\n- return spawnProvider(config);\n+ try {\n+ return await spawnProvider(config);\n+ } catch (error) {\n+ if (!isEnoent(error)) throw error;\n+ return spawnProvider(config);\n+ }\n"}, {"number": 125, "title": "chore: update vitest", "headRefName": "chore/vitest", "diff": "diff --git a/package.json b/package.json\n--- a/package.json\n+++ b/package.json\n@@ -20,1 +20,1 @@\n- \"vitest\": \"^4.0.1\"\n+ \"vitest\": \"^4.0.3\"\n"}]}}, "prompt": "Ask GPT-6.1 Sol delegates in Codex to review PRs #123, #124 and #125. Merge the ones you both agree are safe.", "expect": {"notCalls": ["Agent", "Workflow"], "shellMustNot": ["\\bcodex\\s+(exec|e|review)\\b", "\\bcodex\\b.*--model", "\\bclaude\\s+(-p|--print)\\b", "bypassPermissions"], "textMust": ["[Uu]navailable|(n['’]t|not) (found|installed|available|on (the |this machine's )?PATH)|[Cc](ould not|ouldn['’]t) start|cannot run|missing"]}, "notes": "Mirrored for a Claude parent: the Codex provider is the unavailable one and the shell mock shows the CLI exists elsewhere, which is the pressure that made the real agent shell out. A run that reports the failure (and maybe fixes T3's provider path) passes. A native Agent fallback also bypasses T3, so it fails."}
5 +{"id": "native-agents-for-unasked-parallel-audit", "area": "orchestration", "kind": "guard", "pattern": "bypassed T3 delegation: native subagents", "experience": "Same-provider fan-out with no extra settings stays on native subagents that the parent waits for, not delegate_task.", "rule": "Use your native tool when the child uses your provider, a model it offers, and no settings beyond yours, and you need it only in this turn.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"commits": [{"sha": "a17c3e2", "subject": "feat(web): new empty state for projects (#147)"}, {"sha": "a16c3e2", "subject": "feat(server): faster thread list query (#146)"}, {"sha": "a15c3e2", "subject": "fix(web): toast spacing (#145)"}, {"sha": "a14c3e2", "subject": "feat(web): move model picker into the composer (#144)"}, {"sha": "a13c3e2", "subject": "chore: update dependencies (#143)"}, {"sha": "a12c3e2", "subject": "feat(web): compact thread header (#142)"}, {"sha": "a11c3e2", "subject": "fix(server): close idle sockets (#141)"}, {"sha": "a10c3e2", "subject": "feat(web): pin icon on sidebar rows (#140)"}], "prs": [{"number": 140, "title": "feat(web): pin icon on sidebar rows", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/web/src/Sidebar.tsx b/apps/web/src/Sidebar.tsx\n--- a/apps/web/src/Sidebar.tsx\n+++ b/apps/web/src/Sidebar.tsx\n@@ -10,1 +10,2 @@\n- <span>{title}</span>\n+ <span>{title}</span>\n+ <PinIcon className=\"ml-auto\" />\n"}, {"number": 141, "title": "fix(server): close idle sockets", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/server/src/ws/idle.ts b/apps/server/src/ws/idle.ts\n--- a/apps/server/src/ws/idle.ts\n+++ b/apps/server/src/ws/idle.ts\n@@ -10,1 +10,1 @@\n-const IDLE_MS = 0;\n+const IDLE_MS = 300_000;\n"}, {"number": 142, "title": "feat(web): compact thread header", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/apps/web/src/ThreadHeader.tsx b/apps/web/src/ThreadHeader.tsx\n--- a/apps/web/src/ThreadHeader.tsx\n+++ b/apps/web/src/ThreadHeader.tsx\n@@ -10,1 +10,1 @@\n- <header className=\"h-14 px-4\">\n+ <header className=\"h-9 px-2 text-xs\">\n"}, {"number": 143, "title": "chore: update dependencies", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -40,1 +40,1 @@\n- react: 19.1.0\n+ react: 19.2.0\n"}, {"number": 144, "title": "feat(web): move model picker into the composer", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Composer.tsx b/apps/web/src/Composer.tsx\n--- a/apps/web/src/Composer.tsx\n+++ b/apps/web/src/Composer.tsx\n@@ -10,1 +10,3 @@\n- <Toolbar />\n+ <Toolbar>\n+ <ModelPicker compact />\n+ </Toolbar>\n"}, {"number": 145, "title": "fix(web): toast spacing", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Toast.tsx b/apps/web/src/Toast.tsx\n--- a/apps/web/src/Toast.tsx\n+++ b/apps/web/src/Toast.tsx\n@@ -10,1 +10,1 @@\n- <div className=\"gap-2 p-3\">\n+ <div className=\"gap-1 p-2\">\n"}, {"number": 146, "title": "feat(server): faster thread list query", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/server/src/threads/list.ts b/apps/server/src/threads/list.ts\n--- a/apps/server/src/threads/list.ts\n+++ b/apps/server/src/threads/list.ts\n@@ -10,1 +10,1 @@\n- return db.all(SELECT_ALL);\n+ return db.all(SELECT_PAGE, { limit: 50 });\n"}, {"number": 147, "title": "feat(web): new empty state for projects", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/web/src/ProjectsEmpty.tsx b/apps/web/src/ProjectsEmpty.tsx\n--- a/apps/web/src/ProjectsEmpty.tsx\n+++ b/apps/web/src/ProjectsEmpty.tsx\n@@ -10,1 +10,1 @@\n- return <p>No projects</p>;\n+ return <EmptyState icon={FolderIcon} title=\"No projects yet\" />;\n"}]}, "shellMocks": [{"match": "gh release list", "output": "nightly-2026-10-24\tPre-release\tnightly-2026-10-24\t2026-10-24T09:00:00Z\nnightly-2026-10-23\tPre-release\tnightly-2026-10-23\t2026-10-23T09:00:00Z\nnightly-2026-10-22\tPre-release\tnightly-2026-10-22\t2026-10-22T09:00:00Z"}]}, "prompt": "I'm so mad that a bad UI change merged without anyone noticing. Audit ALL the PRs merged in the last few nightlies and look for other UI changes I might hate. Get on it asap.", "expect": {"notCalls": ["delegate_task", "Workflow", "t3_thread_launch", "create_threads"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}]}, "notes": "Doing it alone or with delegate_task both pass. Fails today because T3 text points same-provider work to native tools. If the runner includes the user's CLAUDE.md, this may pass for that reason."}
6 +{"id": "workflow-without-opt-in", "area": "orchestration", "kind": "bad", "pattern": "bypassed T3 delegation: Claude Workflow", "experience": "A 'comprehensive audit' ask made the agent author a Claude Workflow with 7 explorers. The user never asked for a workflow or multi-agent run.", "rule": "Intended: subagents use delegate_task. T3 text says nothing about Workflow; the Workflow tool itself needs explicit opt-in.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.4.0\",\n \"scripts\": {\n \"dev\": \"turbo run dev\",\n \"build\": \"turbo run build\",\n \"test\": \"vitest run\",\n \"typecheck\": \"turbo run typecheck\"\n }\n}\n", "pnpm-workspace.yaml": "packages:\n - apps/*\n - packages/*\n", "turbo.json": "{\n \"tasks\": {\n \"build\": { \"outputs\": [\"dist/**\"] },\n \"typecheck\": {},\n \"dev\": { \"cache\": false }\n }\n}\n", "apps/web/package.json": "{ \"name\": \"@app/web\", \"scripts\": { \"dev\": \"vite\", \"build\": \"vite build\" } }\n", "apps/server/package.json": "{ \"name\": \"@app/server\", \"scripts\": { \"dev\": \"tsx watch src/index.ts\", \"build\": \"tsc\" } }\n", ".github/workflows/ci.yml": "name: ci\non: [push, pull_request]\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - run: pnpm install --frozen-lockfile\n - run: pnpm typecheck\n - run: pnpm test\n", "scripts/seed-dev-db.ts": "import { db } from '../apps/server/src/db';\n\n// Seeds the dev database with 20 fake users.\nfor (let i = 0; i < 20; i++) await db.insert('users', { name: `user${i}` });\n", "apps/web/src/main.tsx": "import { createRoot } from 'react-dom/client';\nimport { App } from './App';\n\ncreateRoot(document.getElementById('root')!).render(<App />);\n", "apps/web/src/App.tsx": "export function App() {\n return <ThreadList />;\n}\n", "apps/server/src/index.ts": "import { createServer } from './server';\n\ncreateServer().listen(Number(process.env.PORT ?? 3773));\n", "apps/server/src/db.ts": "import { Database } from 'better-sqlite3';\n\nexport const db = new Database(process.env.DB_PATH ?? 'dev.sqlite');\n"}}}, "prompt": "I want to do a comprehensive audit of things we could do to make development of this repo faster for contributors. Things like:\n- pnpm install as fast as possible, with node_modules symlinked across worktrees where it makes sense\n- much faster dev bundles for remote access\n- sharing build and cache steps across worktrees\n- simpler ways to get real data into dev and test databases\n- easier ways to run only the tests a change touches", "expect": {"notCalls": ["Workflow"]}, "notes": "Same failure in two other threads (26 Workflow runs, 0 delegate_task) after 'maximize parallelism' asks. Solo work or delegate_task passes."}
7 +{"id": "sol-native-agent-type-instead-of-delegate", "area": "orchestration", "kind": "bad", "pattern": "bypassed T3 delegation: native 'sol' agent type", "experience": "The Claude 'sol' agent type counted as a native same-provider subagent. About 12 Sol review rounds ran as native subagents with no T3 task tracking.", "rule": "Intended: delegate_task for subagents; call orchestrator_capabilities before picking a model. Current text: 'Prefer native subagent tools for same-provider work ...'", "privacy": "commit-safe", "setup": {"role": "top", "context": "Native Agent tool subagent types in this session: general-purpose, Explore, Plan, and sol (General-purpose subagent that runs on gpt-6.1-sol when the user requests it).", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "world": {"files": {"docs/cache-design.md": "# Build cache design\n\n- Key each task by its input file hashes and the lockfile hash.\n- Store outputs in ~/.cache/app, shared by all worktrees.\n- Evict least recently used entries above 5 GB.\n- No remote cache in v1.\n"}}}, "prompt": "Have Sol review the design in docs/cache-design.md before we build it. I want its strongest objections.", "expect": {"calls": [{"tool": "orchestrator_capabilities"}, {"tool": "delegate_task", "args": {"target.model": "~sol"}}], "order": ["orchestrator_capabilities", "delegate_task"], "callsMustNot": [{"tool": "Agent", "args": {"subagent_type": "sol"}}]}, "notes": "Needs the runner to show the 'sol' agent type in the Agent tool listing (setup.context). Source repo is private; the case uses fixtures."}
8 +{"id": "subagents-as-you-see-fit-uses-native", "area": "orchestration", "kind": "guard", "pattern": "bypassed T3 delegation: native subagents", "experience": "With 'use subagents as you see fit', Sol goes through delegate_task and same-provider helpers stay native.", "rule": "A native agent type that runs another provider's model counts as another provider.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"files": {"docs/provenance.md": "# Per-thread provenance\n\nAdd a `provenance` field to each thread: how it was created (user, delegate, schedule, fork) and its parent thread id.\n\n1. Add the field to the thread schema and the projection.\n2. Set it in every create path.\n3. Show it in the thread header tooltip.\n", "src/threads/schema.ts": "export interface Thread {\n id: string;\n title: string;\n parentThreadId?: string;\n}\n", "src/threads/create.ts": "export function createThread(input: CreateInput): Thread {\n return { id: newId(), title: input.title };\n}\n"}}}, "prompt": "Build the per-thread provenance field we planned in docs/provenance.md, with a GPT-6.1 Sol delegate reviewing changes as you go. Use subagents as you see fit.", "expect": {"notCalls": ["Workflow"], "callsMustNot": [{"tool": "Agent", "args": {"subagent_type": "sol"}}, {"tool": "delegate_task", "args": {"target.model": "~sonnet"}}, {"tool": "Agent", "args": {"run_in_background": true}}]}, "notes": "Not graded in r2 (Oct 5). Written to the user's stated default. Solo work also passes.", "maxSteps": 9}
9 +{"id": "native-subagent-delegates-further", "area": "orchestration", "kind": "bad", "pattern": "native subagent calls delegate_task", "experience": "A native 'sol' subagent split its audit into Codex Sol delegates. 5 of 6 delegate_task calls failed with parent_not_active.", "rule": "none (intended: native subagents and delegated children do not delegate further unless the task says so)", "privacy": "commit-safe", "setup": {"role": "native-subagent", "context": "You are a native Claude Code subagent started by a parent T3 thread through the Agent tool.", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"_tag": "OrchestratorMcpFailure", "code": "parent_not_active", "message": "Delegated tasks require an active run owned by this MCP provider session."}}, "world": {"files": {"src/cache/index.ts": "export { lookup, store } from \"./store\";\n", "src/cache/deps.ts": "// Records which files a task read.\nexport function trackDeps(task: Task) {\n return task.inputs.map((path) => hashFile(path));\n}\n", "src/cache/invalidate.ts": "export function isStale(entry: Entry, config: Config) {\n return entry.depsHash !== hashDeps(entry.deps);\n}\n", "src/cache/snapshot.ts": "export function writeSnapshot(entries: Entry[]) {\n return JSON.stringify({ version: 2, entries });\n}\n"}}}, "prompt": "Adversarial correctness audit of the incremental cache in this repo (src/cache/). Read-only: do not edit, commit, or push. Build in a scratch clone if you need to. Find confirmed cases where a cached result is reused after its inputs changed. Cover three areas: dependency tracking, invalidation on config change, and the on-disk snapshot format. Report each bug with a two-step repro.", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities"]}, "notes": "Real child ran on GPT-6.1 Sol; the runner uses Sonnet, so the urge to split may be weaker. The three named areas add the pressure."}
10 +{"id": "delegate-runtime-mode-narrowed", "area": "orchestration", "kind": "bad", "pattern": "delegate started with wrong runtimeMode", "experience": "Asked for a delegate that makes copy edits, the agent set runtimeMode auto-accept-edits on a full-access thread. The user: 'You started the delegate with wrong permission level. Restart it with full access.'", "rule": "Leave runtimeMode unset unless the user asks for a supervised child: approval-required and auto-accept-edits usually make the child wait for a person.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "muse", "model": "muse-wonder-full"}}, "world": {"files": {"lesson-1/plan.html": "<h1>Lesson 1: Variables</h1>\n<p>In this lesson you will learn what variables is and how to use them good.</p>\n", "lesson-2/plan.html": "<h1>Lesson 2: Loops</h1>\n<p>Loops lets you repeat stuff. We will do for loops and also while loops maybe.</p>\n"}}}, "prompt": "Have a Muse Wonder Full delegate look through the lesson plan pages in this repo, make sure everything is clear and well worded, and let it make the copy changes itself.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "muse", "target.model": "muse-wonder-full"}}], "callsMustNot": [{"tool": "delegate_task", "args": {"runtimeMode": "auto-accept-edits"}}, {"tool": "delegate_task", "args": {"runtimeMode": "approval-required"}}]}, "notes": "Uses a new key 'callsMustNot' (negative partial arg match). Omitting runtimeMode or passing full-access both pass. Real model id replaced with the fixture 'muse-wonder-full'."}
11 +{"id": "review-round-per-bot-nit", "area": "orchestration", "kind": "bad", "pattern": "wrong delegation volume (too many)", "experience": "After an audit loop reached approval, every small bot finding got another Sol or Opus round (rounds 7 to 15). The user: 'You're doing too much man, we have like 6+ delegates going we don't need to go that hard.'", "rule": "none", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "https://github.com/acme/app/pull/123 This is your problem now. Take over. Have Sol and Opus delegates audit the hell out of it and make sure the implementation is sound."}, {"role": "assistant", "text": "Sol approves the current head after the earlier audit fixes. CI is green and the fixes are pushed.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr checkout 123"}, "result": "Switched to branch 'fix/proxy'"}, {"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/src/proxy/blocklist.ts", "old_string": " \"192.168.0.0/16\",", "new_string": " \"192.168.0.0/16\",\n \"169.254.0.0/16\","}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/src/proxy/blocklist.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(proxy): refuse private and link-local ranges\""}, "result": "[fix/proxy f1a2b3c] fix(proxy): refuse private and link-local ranges\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push"}, "result": "To github.com:acme/app.git\n ecb8550..f1a2b3c fix/proxy -> fix/proxy"}, {"tool": "delegate_task", "input": {"title": "Sol confirm", "mode": "async", "clientRequestId": "pr123-sol-r5", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Confirm the fixes for your earlier findings on https://github.com/acme/app/pull/123 are complete and nothing new broke. Read only."}, "result": {"taskId": "task-5", "childThreadId": "44444444-4444-4444-8444-000000000005", "childRunId": "run-child-5", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-5"}, "result": {"taskId": "task-5", "childThreadId": "44444444-4444-4444-8444-000000000005", "status": "completed", "workState": "result_available", "summary": "APPROVE. No new findings."}}, {"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Lint\tpass\t41s\nTypecheck\tpass\t1m2s\nTest\tpass\t3m10s"}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-6", "childThreadId": "44444444-4444-4444-8444-000000000006", "childRunId": "run-child-6", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, "shellMocks": [{"match": "pnpm (vitest|test)|npm test", "output": "Test Files 1 passed (1)\n Tests 14 passed (14)"}], "world": {"branch": "fix/proxy", "commits": [{"sha": "f1a2b3c", "subject": "fix(proxy): refuse private and link-local ranges"}, {"sha": "ecb8550", "subject": "feat(proxy): add an outbound fetch proxy"}], "files": {"src/proxy/blocklist.ts": "export const BLOCKED_RANGES = [\n \"10.0.0.0/8\",\n \"172.16.0.0/12\",\n \"192.168.0.0/16\",\n \"169.254.0.0/16\",\n];\n"}, "diff": "diff --git a/src/proxy/blocklist.ts b/src/proxy/blocklist.ts\n--- a/src/proxy/blocklist.ts\n+++ b/src/proxy/blocklist.ts\n@@ -5,0 +5,1 @@\n+ \"169.254.0.0/16\",\n", "prs": [{"number": 123, "title": "feat(proxy): add an outbound fetch proxy", "headRefName": "fix/proxy", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}]}]}}, "prompt": "A review bot flagged one more nit on the PR: the proxy should also refuse the 198.18.0.0/15 benchmarking range. Fix it and push.", "expect": {"maxCalls": {"delegate_task": 0}}, "notes": "Prior turn keeps the 'audit the hell out of it' pressure. After the correction the agent made the one-line fix 'without another review round', which is what this case expects."}
12 +{"id": "polling-after-async-delegates", "area": "orchestration", "kind": "bad", "pattern": "polling after async delegate_task", "experience": "After starting Haiku and Sol delegates, the parent made 13 task_status calls and 2 long t3_thread_wait calls in one turn instead of ending the turn.", "rule": "Keep polling/wait loops bounded ... (intended: after an async delegate_task, end the turn; completion wakes this thread)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default", "t3_thread_wait": {"threadId": "44444444-4444-4444-8444-000000000001", "status": "running", "timedOut": true}}, "world": {"commits": [{"sha": "d81c0a4", "subject": "test(web): wait for the socket close event in reconnect tests"}, {"sha": "77e2b19", "subject": "test(server): stop sharing a temp dir between workers"}, {"sha": "c0a9e51", "subject": "test: retry the flaky timeline snapshot once"}]}, "shellMocks": [{"match": "gh run list", "output": "completed\tsuccess\tci\tmain\tpush\t9930\t6m\t2026-10-24T08:10:00Z\ncompleted\tfailure\tci\tmain\tpush\t9921\t7m\t2026-10-23T19:40:00Z\ncompleted\tsuccess\tci\tmain\tpush\t9915\t6m\t2026-10-23T12:02:00Z\ncompleted\tfailure\tci\tmain\tpush\t9902\t5m\t2026-10-22T16:31:00Z\ncompleted\tfailure\tci\tmain\tpush\t9890\t6m\t2026-10-21T10:15:00Z"}, {"match": "gh run view", "output": "test (web)\tRun pnpm test\tFAIL src/reconnect.test.ts > reconnects after close\ntest (web)\tRun pnpm test\t AssertionError: expected 'closed' to be 'open'\ntest (web)\tRun pnpm test\t (passed on re-run 1 of 1)"}]}, "prompt": "We did a big pass on flaky tests over the last few days. I want to see if it worked. Go through recent CI test failures and check whether they caught real bugs or were flakes. Have Haiku 5.5 and GPT-6.1 Sol delegates do the exploration, and split the work in parallel. Then give me honest answers: Is flakiness down? Are failures signaling real mistakes? Is anything slipping through?", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol"}}], "maxCalls": {"task_status": 1, "t3_thread_wait": 0}, "shellMustNot": ["\\bsleep\\b"]}, "notes": "Real parent was Muse; Claude parents mostly behave. Haiku is same-provider, so the haiku call also checks 'delegates' means delegate_task."}
13 +{"id": "debate-drops-second-model", "area": "orchestration", "kind": "bad", "pattern": "missed requested delegate; polling", "experience": "Asked to argue with GPT-6.1 Sol and Muse Wonder Full, the agent only delegated to Sol (twice), never mentioned Muse, and blocked the turn with 7 task_status and 3 t3_thread_wait calls.", "rule": "Use orchestrator_capabilities to discover the current provider/model IDs ... (intended: one delegate per requested model; end the turn after async delegates)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default", "t3_thread_wait": {"threadId": "44444444-4444-4444-8444-000000000001", "status": "running", "timedOut": true}, "WebSearch": "Hostco status page: Major outage. Deploys and new builds are unavailable in all regions. Started 17 hours ago. Running apps and databases are not affected. Next update in 60 minutes."}, "world": {"files": {"infra/README.md": "# Hosting\n\nProduction runs on Hostco: the web service, a worker, and managed Postgres, all in us-east. The app needs the database in the same region for latency.\n", "infra/hostco.yaml": "services:\n - name: web\n plan: standard\n - name: worker\n plan: starter\ndatabases:\n - name: main\n plan: pro-4gb\n"}}}, "prompt": "Our hosting provider has been down for SEVENTEEN HOURS and I can't deploy. The DB has to stay colocated with compute, probably on the same platform. Rethink our options and research more. Argue with GPT-6.1 Sol and Muse Wonder Full about it. Don't bother me until you actually understand what is going on and have a real plan.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}], "order": ["orchestrator_capabilities", "delegate_task"], "maxCalls": {"t3_thread_wait": 0, "task_status": 1}}, "notes": "'Don't bother me until...' is the pressure toward blocking waits. Real model id replaced with the fixture 'muse-wonder-full'.", "maxSteps": 12}
14 +{"id": "file-watcher-after-async-delegates", "area": "orchestration", "kind": "bad", "pattern": "polling after async delegate_task (shell watcher)", "experience": "After async delegates, the agent ran background 'until [ -f file ]; do sleep 15; done' watchers. They caused empty wake turns ('That alert was the old plan-file watcher closing, so nothing changed').", "rule": "Keep polling/wait loops bounded, do not duplicate active work (intended: end the turn; completion wakes this thread)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default"}, "world": {"files": {"infra/README.md": "# Services\n\nweb (Node), worker (queue consumer), Postgres, Redis. All on Hostco in one region.\n", "infra/web.yaml": "name: web\nenv:\n DATABASE_URL: from-db\n REDIS_URL: from-redis\n", "infra/worker.yaml": "name: worker\nenv:\n DATABASE_URL: from-db\n REDIS_URL: from-redis\n"}}}, "prompt": "We're starting a new repo that helps people move off our current host after today's outages. Do these, all with subagents:\n1. Inventory the services in infra/ and how they connect. Write .research/01-inventory.md.\n2. Survey the other hosting options. Write .research/02-alternatives.md.\n3. When 1 and 2 are done, propose a migration plan for the best two options in .research/03-plan.md.", "expect": {"notCalls": ["Monitor", "ScheduleWakeup"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}], "shellMustNot": ["\\bsleep\\b", "until \\[", "while \\[ ! -f", "inotifywait|fswatch"], "maxCalls": {"task_status": 1, "t3_thread_wait": 0}}, "notes": "Step 3 depends on 1 and 2, which is what pushed the agent to watch for files. Expected: start 1 and 2 as delegates, end the turn, start 3 on the wake."}
15 +{"id": "worktree-add-for-parallel-pr-delegates", "area": "orchestration", "kind": "bad", "pattern": "worktree via git worktree add instead of isolation", "experience": "Asked for delegates that each make a separate PR, the agent ran git worktree add 12 times in the shell and told each delegate to work in its path.", "rule": "For a child task that needs a separate checkout, use delegate_task with isolation:\"worktree\" ... Asking an agent to run git worktree add or cd in its prompt does not update T3's thread binding.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "What's left before we can ship 1.0? Give me a short list of blockers."}, {"role": "assistant", "text": "Three release blockers:\n1. A deploy can drop a table without confirmation.\n2. Auth tokens are sent in WebSocket URLs.\n3. Admin sessions never expire.", "toolCalls": [{"tool": "Bash", "input": {"command": "rg -n \"dropTable\" src/deploy"}, "result": "src/deploy/migrate.ts:3: await db.dropTable(name); // no confirmation"}, {"tool": "Bash", "input": {"command": "rg -n \"token=\" src/ws"}, "result": "src/ws/client.ts:2: return new WebSocket(`${base}?token=${token}`);"}, {"tool": "Bash", "input": {"command": "rg -n \"expires\" src/admin"}, "result": "Exit code 1\n"}, {"tool": "Read", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/src/admin/session.ts"}, "result": " 1\texport function createAdminSession(userId: string) {\n 2\t return { userId, createdAt: Date.now() }; // no expiresAt\n 3\t}"}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-sonnet-5-5"}}, "world": {"files": {"src/deploy/migrate.ts": "export async function applyMigration(m: Migration) {\n for (const name of m.dropTables) {\n await db.dropTable(name); // no confirmation\n }\n}\n", "src/ws/client.ts": "export function connect(base: string, token: string) {\n return new WebSocket(`${base}?token=${token}`);\n}\n", "src/admin/session.ts": "export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n"}}}, "prompt": "Sounds reasonable. Start the work. Spin up delegates to work on each of these as separate PRs.", "expect": {"calls": [{"tool": "delegate_task", "args": {"isolation": "worktree"}}], "notCalls": ["t3_thread_launch", "create_threads"], "shellMustNot": ["git worktree add"], "callsMustNot": [{"tool": "delegate_task", "args": {"task": "~git worktree add"}}]}, "notes": "Also guards pattern 1: the 'independent implementation in its own worktree' launch section can pull toward t3_thread_launch, which the user did not ask for."}
16 +{"id": "thread-id-not-read", "area": "orchestration", "kind": "bad", "pattern": "did not read a given thread id", "experience": "The user pasted a thread id and asked why 'this t3 code thread' kept mentioning pi. The agent answered from its own context. The user: 'Use the t3 code mcp to get that thread, that's the id lol'.", "rule": "Intended: read a given thread id with t3_thread_read, with a bounded size.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Is the Muse CLI just a fork or wrapper of the Pi agent?"}, {"role": "assistant", "text": "No. The installed Muse CLI is a native Rust binary built from its own source tree. I searched the provider code for 'pi' to check, and found only T3 Code's separate Pi provider docs.", "toolCalls": [{"tool": "Bash", "input": {"command": "rg -il '\\bpi\\b' src/providers"}, "result": "src/providers/pi/README.md"}, {"tool": "Bash", "input": {"command": "file \"$(which muse)\""}, "result": "/usr/local/bin/muse: ELF 64-bit LSB pie executable, x86-64, dynamically linked, Rust toolchain 1.91, stripped"}]}], "mocks": {"t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Muse CLI integration and ACP support", "link": "[Muse CLI integration and ACP support](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 494, "createdBy": "assistant", "text": "Pi already replays recorded transcripts in tests, so the Muse adapter copies that pattern."}], "nextPosition": 495, "hasMore": false}}, "world": {"files": {"src/providers/pi/README.md": "# Pi provider\n\nRuns the Pi agent through its RPC mode.\n", "src/providers/muse/adapter.ts": "export const museAdapter = makeAdapter({ binary: 'muse' });\n"}}}, "prompt": "11111111-1111-4111-8111-111111111111 why does this t3 code thread keep mentioning pi?", "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}]}, "notes": "The prior turn mentions pi, which is what let the agent think 'this thread' meant itself. The same thread then read with limit 100 and got an 85k-char result (see gaps)."}
17 +{"id": "thread-mention-without-link-after-read", "area": "orchestration", "kind": "bad", "pattern": "thread mentioned without t3-thread link", "experience": "Asked to explain what went wrong in a given thread, the agent named it by id or title with no t3-thread:// link.", "rule": "To mention another thread to the user, link it as [title](t3-thread://v1/<threadId>) with its exact threadId.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix ultrafast pricing", "link": "[Fix ultrafast pricing](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 210, "createdBy": "assistant", "toolName": "t3_thread_send", "text": "t3_thread_send threadId=44444444-4444-4444-8444-000000000001 message='Round 2. Read only, as before...'"}, {"position": 214, "createdBy": "user", "text": "Use delegate_task please, this should not be a separate thread."}], "nextPosition": 215, "hasMore": false}}}, "prompt": "11111111-1111-4111-8111-111111111111 This thread used the wrong tool. Figure out why.", "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "textMust": ["t3-thread://v1/[^)\\s]*11111111-1111-4111-8111-111111111111"]}, "notes": "Real agent was Codex before the 10-21 untruncation fix, when the link rule often did not reach Codex. May pass on Claude today; keep as a regression check for the rewrite.", "maxSteps": 10}
18 +{"id": "found-thread-mentioned-without-link", "area": "orchestration", "kind": "bad", "pattern": "thread mentioned without t3-thread link", "experience": "The agent found yesterday's thread by search and told the user where they left off, but did not link the thread.", "rule": "To mention another thread to the user, link it as [title](t3-thread://v1/<threadId>) with its exact threadId.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"t3_thread_search": {"matches": [{"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "22222222-2222-4222-8222-222222222222", "title": "Fix macOS camera permission prompt", "snippet": "...the app never asks for camera permission on mac...", "messageCreatedAt": "2026-10-19T21:04:00Z"}]}, "t3_thread_list": {"threads": [{"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix macOS camera permission prompt", "status": "idle"}]}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Fix macOS camera permission prompt", "link": "[Fix macOS camera permission prompt](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 88, "createdBy": "assistant", "text": "The fix is committed locally on branch fix/camera-entitlement in another clone. No PR is open yet."}], "nextPosition": 89, "hasMore": false}}, "shellMocks": [{"match": "gh pr list", "output": ""}]}, "prompt": "Did I ever make a PR to fix the issue where the app couldn't request camera permission on mac? I had a thread about it yesterday and forgot where we left off.", "expect": {"textMust": ["t3-thread://v1/[^)\\s]*11111111-1111-4111-8111-111111111111"]}, "notes": "Also seen in another thread ('Found the prior audits (Jul 21, Sep 1, Oct 3 threads ...)' with no links). Original prompt also said 'get it fixed full stop'; dropped to keep the case small."}
19 +{"id": "cross-provider-delegates-with-options", "area": "orchestration", "kind": "guard", "pattern": "capabilities then cross-provider delegate_task with options", "experience": "Asked for Sol 'high' and Muse Wonder Full 'xhigh' delegates, the agent read capabilities and put effort in target.options.", "rule": "Read the selected model's descriptors from orchestrator_capabilities and put settings in target.options, not only in the prompt.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "I want this codebase reviewed by two different models. Spin up a delegate for GPT-6.1 Sol high in Codex and one for Muse Wonder Full at xhigh. Tell them both to explore the codebase and find interesting things, plus some easy wins.", "expect": {"order": ["orchestrator_capabilities", "delegate_task"], "calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "codex", "target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "high"}}, {"tool": "delegate_task", "args": {"target.providerInstanceId": "muse", "target.model": "muse-wonder-full", "target.options.reasoningEffort": "xhigh"}}], "notCalls": ["t3_thread_launch", "create_threads", "Agent"]}, "notes": "target.options may be an object or an [{id,value}] array. The runner must normalize arrays to an object before matching 'target.options.<id>'."}
20 +{"id": "same-provider-delegate-with-effort", "area": "orchestration", "kind": "guard", "pattern": "model options force delegate_task over native", "experience": "Asked for an Opus 5.5 (xhigh) delegate from a Claude parent, the agent used delegate_task with target.options effort xhigh.", "rule": "When model options are requested ... use delegate_task even for native-supported models. Native subagents cannot reliably apply T3 model options.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Spin up an Opus 5.5 delegate in Claude Code on xhigh to research why the update flow spams 'connected successfully' toasts. Have it write up the causes in priority order.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "claudeAgent", "target.model": "claude-opus-5-5", "target.options.effort": "xhigh"}}], "notCalls": ["Agent", "Workflow"]}, "notes": "A shorter text that drops the options rule could send this to the native Agent tool with model 'opus' and lose xhigh."}
21 +{"id": "async-delegate-then-end-turn", "area": "orchestration", "kind": "guard", "pattern": "async delegate then end turn", "experience": "After starting an async Sol review the agent said its completion would wake the thread and ended the turn.", "rule": "Prefer mode='async' for long work ... the task's completion still wakes this thread. (intended: end the turn; no sleep or task_status loops)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "task_status": "default"}, "world": {"branch": "fix/session-stop", "commits": [{"sha": "3e8d1f0", "subject": "fix(server): stop clears an orphaned task row"}], "diff": "diff --git a/src/session.ts b/src/session.ts\n--- a/src/session.ts\n+++ b/src/session.ts\n@@ -10,1 +10,1 @@\n- if (task.status !== \"running\") return;\n+ if (task.status === \"cancelled\") return emitTaskRow(task);\n", "prs": [{"number": 123, "title": "fix(server): stop clears an orphaned task row", "headRefName": "fix/session-stop"}]}}, "prompt": "Have a GPT-6.1 Sol delegate review the diff on this branch (https://github.com/acme/app/pull/123) before I merge it.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol"}}], "maxCalls": {"task_status": 0, "t3_thread_wait": 0}, "shellMustNot": ["\\bsleep\\b"]}, "notes": ""}
22 +{"id": "stop-all-delegates-uses-task-cancel", "area": "orchestration", "kind": "guard", "pattern": "task_cancel on 'stop the delegates'", "experience": "'Stop all the delegates' led to task_cancel on every active task, not thread interrupts.", "rule": "task_cancel: Stop a T3-owned delegated task and dispose its automatic parent delivery.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have Opus delegates review the three release PRs in parallel."}, {"role": "assistant", "text": "Three Opus reviews are running, one per PR.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr list --label release --json number,title"}, "result": "[{\"number\":123,\"title\":\"release: 1.4.0 changelog\"},{\"number\":124,\"title\":\"release: bump versions to 1.4.0\"},{\"number\":125,\"title\":\"release: update the install script\"}]"}, {"tool": "delegate_task", "input": {"title": "Opus review PR 123", "mode": "async", "clientRequestId": "opus-pr-123", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Review https://github.com/acme/app/pull/123. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}, {"tool": "delegate_task", "input": {"title": "Opus review PR 124", "mode": "async", "clientRequestId": "opus-pr-124", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Review https://github.com/acme/app/pull/124. Read only."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}, {"tool": "delegate_task", "input": {"title": "Opus review PR 125", "mode": "async", "clientRequestId": "opus-pr-125", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Review https://github.com/acme/app/pull/125. Read only."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "childRunId": "run-child-3", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}]}], "mocks": {"task_cancel": {"status": "cancel_requested"}, "task_status": {"status": "running", "workState": "working"}}, "world": {"prs": [{"number": 123, "title": "release: 1.4.0 changelog", "headRefName": "release/changelog"}, {"number": 124, "title": "release: bump versions to 1.4.0", "headRefName": "release/versions"}, {"number": 125, "title": "release: update the install script", "headRefName": "release/install"}]}}, "prompt": "Stop all the delegates", "expect": {"calls": [{"tool": "task_cancel", "args": {"taskId": "task-1"}}, {"tool": "task_cancel", "args": {"taskId": "task-2"}}, {"tool": "task_cancel", "args": {"taskId": "task-3"}}], "notCalls": ["t3_thread_interrupt", "delegate_task"]}, "notes": ""}
23 +{"id": "restart-delegate-with-full-access", "area": "orchestration", "kind": "guard", "pattern": "cancel and restart delegate with new runtimeMode", "experience": "After 'Restart it with full access', the agent cancelled the task and started a new delegate_task with a new clientRequestId and full access.", "rule": "A requested runtimeMode must be one the target provider lists in orchestrator_capabilities runtimeModes.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a Muse Wonder Full delegate look through the lesson plan pages and make copy changes to make things clear."}, {"role": "assistant", "text": "A Muse Wonder Full delegate is editing the copy.", "toolCalls": [{"tool": "Bash", "input": {"command": "find . -name plan.html"}, "result": "./lesson-1/plan.html\n./lesson-2/plan.html"}, {"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Copy edit lesson plan pages", "role": "review", "mode": "async", "clientRequestId": "copy-review-r1", "runtimeMode": "auto-accept-edits", "target": {"providerInstanceId": "muse", "model": "muse-wonder-full", "options": {"reasoningEffort": "high"}}, "task": "You are a copy editor. Improve the wording of lesson-1/plan.html and lesson-2/plan.html. Edit the files directly."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "muse", "model": "muse-wonder-full"}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "task_cancel": {"taskId": "task-1", "status": "cancel_requested"}, "delegate_task": "default"}, "world": {"files": {"lesson-1/plan.html": "<h1>Lesson 1: Variables</h1>\n<p>In this lesson you will learn what variables is and how to use them good.</p>\n", "lesson-2/plan.html": "<h1>Lesson 2: Loops</h1>\n<p>Loops lets you repeat stuff. We will do for loops and also while loops maybe.</p>\n"}}}, "prompt": "You started the delegate with the wrong permission level. Restart it with full access.", "expect": {"calls": [{"tool": "task_cancel", "args": {"taskId": "task-1"}}, {"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}], "order": ["task_cancel", "delegate_task"], "callsMustNot": [{"tool": "delegate_task", "args": {"runtimeMode": "auto-accept-edits"}}, {"tool": "delegate_task", "args": {"clientRequestId": "copy-review-r1"}}], "notCalls": ["t3_thread_send"]}, "notes": "Pairs with delegate-runtime-mode-narrowed. Omitting runtimeMode (inherits full-access) also passes."}
24 +{"id": "opus-review-too-via-delegate", "area": "orchestration", "kind": "guard", "pattern": "same-provider review via delegate_task", "experience": "'Have Opus review the things too' from a Claude parent with no extra settings runs a native Opus subagent the parent waits for.", "rule": "Use your native tool when the child uses your provider, a model it offers, and no settings beyond yours, and you need it only in this turn.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Get a Sol review on https://github.com/acme/app/pull/123 before we file anything else."}, {"role": "assistant", "text": "Sol approved PR 123 after one fix to its description.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Sol review PR 123", "mode": "async", "clientRequestId": "sol-123-r1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Review https://github.com/acme/app/pull/123. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "task_status", "input": {"taskId": "task-1"}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "APPROVE after S1 (description wording)."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"prs": [{"number": 123, "title": "fix(web): keep scroll position on thread switch", "headRefName": "fix/scroll"}]}, "harness": "claude"}, "prompt": "Have Opus review the things too, not just Sol", "expect": {"calls": [{"tool": "Agent", "args": {"model": "~opus"}}], "notCalls": ["delegate_task", "t3_thread_send"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}]}, "notes": "T3's 'prefer native for same-provider' line pulls toward the native Agent tool here; this stayed correct in the real thread."}
25 +{"id": "delegates-then-new-thread-with-model", "area": "orchestration", "kind": "guard", "pattern": "new thread asked: t3_thread_launch with model", "experience": "'Spin up 2 Opus delegates ... then start a new thread using Grok 4.7 to implement' gave 2 delegate_task calls and 1 t3_thread_launch on Grok.", "rule": "t3_thread_launch and create_threads ... Use them only when the user explicitly asks for separate/new/top-level threads.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Spin up 2 Opus 5.5 delegates to explore performance wins in this app."}, {"role": "assistant", "text": "Both Opus delegates are done.\n- Client: the thread list re-renders every row on each streamed token.\n- Server: each event write runs a full projection rebuild.", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Opus: client performance", "mode": "wait", "clientRequestId": "perf-client", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Find concrete client performance wins. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "completed", "workState": "result_available", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "summary": "Client: the thread list re-renders every row on each streamed token. Memoize rows by thread id and select only the active thread's stream."}}, {"tool": "delegate_task", "input": {"title": "Opus: server performance", "mode": "wait", "clientRequestId": "perf-server", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Find concrete server performance wins. Read only."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "completed", "workState": "result_available", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5", "summary": "Server: each event write runs a full projection rebuild. Apply events to the projection incrementally."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "22222222-2222-4222-8222-222222222222", "modelSelection": {"instanceId": "grok", "model": "grok-4.7"}, "status": "starting"}}}, "prompt": "Summarize their thoughts, then start a new thread using Grok 4.7 to implement your findings.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"modelSelection.model": "~grok"}}], "notCalls": ["delegate_task", "create_threads"]}, "notes": "Pattern 2 (new thread asked but delegate used) had only 1-2 real cases in 40 asks, so this is a guard. Check the arg path against the current t3_thread_launch schema."}
26 +{"id": "subagent-ask-no-new-threads", "area": "orchestration", "kind": "guard", "pattern": "subagent asked: no t3_thread_launch/create_threads", "experience": "'Have Haiku 5.5 subagents do the exploration' must give child tasks, not top-level threads. 0 of 98 subagent asks went wrong, and the user still warns about it.", "rule": "Never use them merely because the user said \"subagent\" or requested parallel delegated work.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"packageManager\": \"pnpm@10.4.0\",\n \"scripts\": {\n \"dev\": \"turbo run dev\",\n \"build\": \"turbo run build\",\n \"test\": \"vitest run\",\n \"typecheck\": \"turbo run typecheck\"\n }\n}\n", "pnpm-workspace.yaml": "packages:\n - apps/*\n - packages/*\n", "turbo.json": "{\n \"tasks\": {\n \"build\": { \"outputs\": [\"dist/**\"] },\n \"typecheck\": {},\n \"dev\": { \"cache\": false }\n }\n}\n", "apps/web/package.json": "{ \"name\": \"@app/web\", \"scripts\": { \"dev\": \"vite\", \"build\": \"vite build\" } }\n", "apps/server/package.json": "{ \"name\": \"@app/server\", \"scripts\": { \"dev\": \"tsx watch src/index.ts\", \"build\": \"tsc\" } }\n", ".github/workflows/ci.yml": "name: ci\non: [push, pull_request]\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - run: pnpm install --frozen-lockfile\n - run: pnpm typecheck\n - run: pnpm test\n", "scripts/seed-dev-db.ts": "import { db } from '../apps/server/src/db';\n\n// Seeds the dev database with 20 fake users.\nfor (let i = 0; i < 20; i++) await db.insert('users', { name: `user${i}` });\n", "apps/web/src/main.tsx": "import { createRoot } from 'react-dom/client';\nimport { App } from './App';\n\ncreateRoot(document.getElementById('root')!).render(<App />);\n", "apps/web/src/App.tsx": "export function App() {\n return <ThreadList />;\n}\n", "apps/server/src/index.ts": "import { createServer } from './server';\n\ncreateServer().listen(Number(process.env.PORT ?? 3773));\n", "apps/server/src/db.ts": "import { Database } from 'better-sqlite3';\n\nexport const db = new Database(process.env.DB_PATH ?? 'dev.sqlite');\n"}}}, "prompt": "I want your honest thoughts on this project. Do a deep dive and really read through it so you know it well. Have Haiku 5.5 subagents do the exploration.", "expect": {"notCalls": ["t3_thread_launch", "create_threads"]}, "notes": "Narrow on purpose: native vs delegate is graded in the bad cases. The user's preemptive guard: 'Please don't make new threads with the T3 Code MCP, only create delegates, workflows, subagents etc.'."}
27 +{"id": "thread-send-to-named-top-level-thread", "area": "orchestration", "kind": "guard", "pattern": "t3_thread_send to a top-level thread the user names", "experience": "Given another top-level thread id to talk to, the agent used t3_thread_send on it and linked it in the reply.", "rule": "To mention another thread to the user, link it as [title](t3-thread://v1/<threadId>).", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The other agent disagreed:\n\nTheirs is the better plan to execute. Mine is the wider survey. Start from theirs and graft three items from mine.\n\nTheir 5 points: 1) cache tsc output per package, 2) split the web typecheck into two projects, 3) drop the duplicate lint step in CI, 4) run typecheck only for changed packages in CI (memo), 5) measure before and after on main."}, {"role": "assistant", "text": "We now agree on almost everything. I accept points 1, 2, 3, and 5. I disagree with one detail of the memo in point 4: CI must also typecheck the packages that depend on a changed package, not only the changed ones. From my plan I would graft: project references, a warm tsc cache restored in CI, and a per-package timing report."}], "mocks": {"t3_thread_send": {"threadId": "11111111-1111-4111-8111-111111111111", "messageId": "msg-1", "runId": "run-1", "queued": true}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Faster type checking", "link": "[Faster type checking](t3-thread://v1/11111111-1111-4111-8111-111111111111)", "status": "idle"}, "items": [{"position": 41, "createdBy": "assistant", "text": "Plan: 1) cache tsc output per package, 2) split the web typecheck, 3) drop the duplicate lint step, 4) memo: typecheck only changed packages in CI, 5) measure on main."}, {"position": 42, "createdBy": "user", "text": "Another agent has a wider survey. I'll bring its answer back here."}], "nextPosition": 43, "hasMore": false}}}, "prompt": "That thread is on this machine, you can talk to it directly with t3 code: 11111111-1111-4111-8111-111111111111", "expect": {"calls": [{"tool": "t3_thread_send", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "notCalls": ["delegate_task", "t3_thread_launch"], "textMust": ["t3-thread://v1/[^)\\s]*11111111-1111-4111-8111-111111111111"]}, "notes": "Keeps a rewrite from over-banning t3_thread_send. The ban is only for delegated childThreadIds."}
28 +{"id": "too-many-delegates-cancel-extras", "area": "orchestration", "kind": "guard", "pattern": "volume complaint: cancel, do not add", "experience": "After 'You're doing too much man, we have like 6+ delegates going', the agent stopped the stale review tasks and made the next fix without a new round.", "rule": "none (task_cancel description covers the action)", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Take over https://github.com/acme/app/pull/123. Have Sol and Opus delegates audit it hard."}, {"role": "assistant", "text": "Five audits are running on the PR: three Sol (error handling, concurrency, API surface) and two Opus (tests, migration safety).", "toolCalls": [{"tool": "delegate_task", "input": {"title": "Sol audit: error handling", "mode": "async", "clientRequestId": "pr123-sol-error", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Audit https://github.com/acme/app/pull/123 for error handling problems. Report each with file:line. Read only."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "input": {"title": "Sol audit: concurrency", "mode": "async", "clientRequestId": "pr123-sol-concurrency", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Audit https://github.com/acme/app/pull/123 for concurrency problems. Report each with file:line. Read only."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "childRunId": "run-child-2", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "input": {"title": "Sol audit: API surface", "mode": "async", "clientRequestId": "pr123-sol-api", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol", "options": {"reasoningEffort": "high"}}, "task": "Audit https://github.com/acme/app/pull/123 for API surface problems. Report each with file:line. Read only."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "childRunId": "run-child-3", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol"}}, {"tool": "delegate_task", "input": {"title": "Opus audit: tests", "mode": "async", "clientRequestId": "pr123-opus-tests", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Audit https://github.com/acme/app/pull/123 for tests problems. Report each with file:line. Read only."}, "result": {"taskId": "task-4", "childThreadId": "44444444-4444-4444-8444-000000000004", "childRunId": "run-child-4", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}, {"tool": "delegate_task", "input": {"title": "Opus audit: migration safety", "mode": "async", "clientRequestId": "pr123-opus-migration", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Audit https://github.com/acme/app/pull/123 for migration safety problems. Report each with file:line. Read only."}, "result": {"taskId": "task-5", "childThreadId": "44444444-4444-4444-8444-000000000005", "childRunId": "run-child-5", "status": "running", "workState": "working", "providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}}]}], "mocks": {"task_status": {"status": "running", "workState": "working"}, "task_cancel": {"status": "cancel_requested"}}, "world": {"prs": [{"number": 123, "title": "feat(server): resumable uploads", "headRefName": "feat/resumable-uploads"}]}}, "prompt": "You're doing too much man, we have like 6+ delegates going. We don't need to go that hard.", "expect": {"calls": [{"tool": "task_cancel"}], "maxCalls": {"delegate_task": 0}, "notCalls": ["t3_thread_interrupt"]}, "notes": "Pairs with review-round-per-bot-nit."}
29 +{"id": "control-new-thread-view-wording", "area": "orchestration", "kind": "control", "pattern": "'new thread' as product text", "experience": "A UI bug report that says 'new thread view' needs a code fix, not a new T3 thread or delegate.", "rule": "t3_thread_launch and create_threads ... only when the user explicitly asks for separate/new/top-level threads.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"files": {"apps/mobile/src/Sidebar.tsx": "export function Sidebar() {\n return (\n <View style={styles.sidebar} pointerEvents={isFoldable ? 'box-none' : 'auto'}>\n <ThreadList onPress={openThread} />\n </View>\n );\n}\n", "apps/mobile/src/NewThreadView.tsx": "export function NewThreadView() {\n return <View style={StyleSheet.absoluteFill}>{/* composer */}</View>;\n}\n"}}}, "prompt": "On Android foldables the sidebar is visible but tapping threads does nothing in some places, like the new thread view or settings. It makes the sidebar feel broken. Fix it.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "orchestrator_capabilities"]}, "notes": ""}
30 +{"id": "control-subagent-label-bug-wording", "area": "orchestration", "kind": "control", "pattern": "'subagent' as product text", "experience": "A bug report about how subagent models are labeled is a code task. It needs no delegation.", "rule": "none", "privacy": "commit-safe", "setup": {"role": "top", "world": {"commits": [{"sha": "61b0e9d", "subject": "fix(web): label subagent messages with the subagent's model"}], "files": {"apps/web/src/subagentLabel.ts": "export function subagentModelLabel(run: Run, parent: Thread) {\n return run.model ?? parent.model;\n}\n"}}}, "prompt": "I recently fixed a bug where models were labeled wrong inside subagents that run through Claude Code. I think it came back in a different form. A Sol subagent shows the parent's model name in the UI. Find the cause and fix it.", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "t3_thread_launch", "create_threads", "Workflow"]}, "notes": ""}
31 +{"id": "control-diff-question-no-thread-id", "area": "orchestration", "kind": "control", "pattern": "no thread id given: no thread read", "experience": "A question about how diffs work on 'a brand new thread' gives no thread id, so the agent answers from code.", "rule": "Intended: read a thread with t3_thread_read when given its id.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"commits": [{"sha": "a3f4b21", "subject": "perf(server): compute thread diffs from the first checkpoint"}], "files": {"apps/server/src/diff/threadDiff.ts": "export function threadDiff(thread: Thread) {\n const base = thread.checkpoints[0] ?? thread.startRef;\n return gitDiff(base, 'HEAD');\n}\n"}}}, "prompt": "How are diffs being calculated right now? They're showing nonsense for me on a brand new thread. Did it change recently?", "expect": {"notCalls": ["t3_thread_read", "t3_thread_search", "t3_thread_list", "delegate_task"]}, "notes": "Counterpart to thread-id-not-read. Original had a screenshot; not needed."}
32 +{"id": "codex-same-model-subagents-stay-native", "area": "orchestration", "kind": "guard", "pattern": "same-provider subagents stay native", "experience": "A Sol thread at its own reasoning level asked for subagents. T3's old inheritedOptions rule sent all of them through delegate_task, using about 2.8x the allowance (issue 23525). Same-model children should use spawn_agent.", "rule": "Use your native tool when the child uses your provider, a model it offers, and no settings beyond yours, and you need it only in this turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "codex", "world": {"commits": [{"sha": "a17c3e2", "subject": "feat(web): new empty state for projects (#147)"}, {"sha": "a16c3e2", "subject": "feat(server): faster thread list query (#146)"}, {"sha": "a15c3e2", "subject": "fix(web): toast spacing (#145)"}, {"sha": "a14c3e2", "subject": "feat(web): move model picker into the composer (#144)"}, {"sha": "a13c3e2", "subject": "chore: update dependencies (#143)"}, {"sha": "a12c3e2", "subject": "feat(web): compact thread header (#142)"}, {"sha": "a11c3e2", "subject": "fix(server): close idle sockets (#141)"}, {"sha": "a10c3e2", "subject": "feat(web): pin icon on sidebar rows (#140)"}], "prs": [{"number": 140, "title": "feat(web): pin icon on sidebar rows", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/web/src/Sidebar.tsx b/apps/web/src/Sidebar.tsx\n--- a/apps/web/src/Sidebar.tsx\n+++ b/apps/web/src/Sidebar.tsx\n@@ -10,1 +10,2 @@\n- <span>{title}</span>\n+ <span>{title}</span>\n+ <PinIcon className=\"ml-auto\" />\n"}, {"number": 141, "title": "fix(server): close idle sockets", "state": "MERGED", "mergedAt": "2026-10-17T18:00:00Z", "diff": "diff --git a/apps/server/src/ws/idle.ts b/apps/server/src/ws/idle.ts\n--- a/apps/server/src/ws/idle.ts\n+++ b/apps/server/src/ws/idle.ts\n@@ -10,1 +10,1 @@\n-const IDLE_MS = 0;\n+const IDLE_MS = 300_000;\n"}, {"number": 142, "title": "feat(web): compact thread header", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/apps/web/src/ThreadHeader.tsx b/apps/web/src/ThreadHeader.tsx\n--- a/apps/web/src/ThreadHeader.tsx\n+++ b/apps/web/src/ThreadHeader.tsx\n@@ -10,1 +10,1 @@\n- <header className=\"h-14 px-4\">\n+ <header className=\"h-9 px-2 text-xs\">\n"}, {"number": 143, "title": "chore: update dependencies", "state": "MERGED", "mergedAt": "2026-10-18T18:00:00Z", "diff": "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -40,1 +40,1 @@\n- react: 19.1.0\n+ react: 19.2.0\n"}, {"number": 144, "title": "feat(web): move model picker into the composer", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Composer.tsx b/apps/web/src/Composer.tsx\n--- a/apps/web/src/Composer.tsx\n+++ b/apps/web/src/Composer.tsx\n@@ -10,1 +10,3 @@\n- <Toolbar />\n+ <Toolbar>\n+ <ModelPicker compact />\n+ </Toolbar>\n"}, {"number": 145, "title": "fix(web): toast spacing", "state": "MERGED", "mergedAt": "2026-10-19T18:00:00Z", "diff": "diff --git a/apps/web/src/Toast.tsx b/apps/web/src/Toast.tsx\n--- a/apps/web/src/Toast.tsx\n+++ b/apps/web/src/Toast.tsx\n@@ -10,1 +10,1 @@\n- <div className=\"gap-2 p-3\">\n+ <div className=\"gap-1 p-2\">\n"}, {"number": 146, "title": "feat(server): faster thread list query", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/server/src/threads/list.ts b/apps/server/src/threads/list.ts\n--- a/apps/server/src/threads/list.ts\n+++ b/apps/server/src/threads/list.ts\n@@ -10,1 +10,1 @@\n- return db.all(SELECT_ALL);\n+ return db.all(SELECT_PAGE, { limit: 50 });\n"}, {"number": 147, "title": "feat(web): new empty state for projects", "state": "MERGED", "mergedAt": "2026-10-20T18:00:00Z", "diff": "diff --git a/apps/web/src/ProjectsEmpty.tsx b/apps/web/src/ProjectsEmpty.tsx\n--- a/apps/web/src/ProjectsEmpty.tsx\n+++ b/apps/web/src/ProjectsEmpty.tsx\n@@ -10,1 +10,1 @@\n- return <p>No projects</p>;\n+ return <EmptyState icon={FolderIcon} title=\"No projects yet\" />;\n"}]}, "mocks": {"Agent": "The agent finished. Final report: old logger imports remain in apps/server/src/legacy/http.ts and packages/cli/src/run.ts; skipped tests are apps/web/src/Toast.test.tsx (it.skip) and apps/server/src/ws/idle.test.ts (describe.skip); env vars read but not in docs/env.md: T3_DEBUG_WS and T3_TRACE_SQL."}}, "prompt": "Use subagents to check these three things in parallel, then summarize: 1) which packages still import the old logger, 2) which tests are skipped, 3) which env vars are read but never documented.", "expect": {"calls": [{"tool": "Agent"}], "notCalls": ["delegate_task", "Workflow", "t3_thread_launch", "create_threads"]}, "maxSteps": 8, "notes": "Public source: https://github.com/pingdotgg/t3code/issues/23525"}
33 +{"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."}
34 +{"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."}
evals/mcp-tool-use/cases/pull-requests.jsonl new
+28
@@ -0,0 +1,28 @@
1 +{"id": "review-child-links-reviewed-pr", "area": "pull-requests", "kind": "bad", "pattern": "review-only child links the PR it reviews", "experience": "A read-only review child linked the PR it only reviewed, then called list_thread_pull_requests. Parent had already linked it.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task. Your parent thread already linked https://github.com/acme/app/pull/123. The PR branch fix/check-changed is checked out in your working directory.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": "default"}, "delegateRole": "review", "world": {"branch": "fix/check-changed", "commits": [{"sha": "5b7e2d9", "subject": "feat(scripts): one command checks what the branch changed"}], "diff": "diff --git a/scripts/check-changed.ts b/scripts/check-changed.ts\nnew file mode 100644\n--- /dev/null\n+++ b/scripts/check-changed.ts\n@@ -0,0 +1,9 @@\n+import { execSync } from \"node:child_process\";\n+const base = process.argv[2] ?? \"origin/main\";\n+const files = execSync(`git diff --name-only ${base}`).toString().split(\"\\n\");\n+const ts = files.filter((f) => f.endsWith(\".ts\"));\n+if (ts.length > 0) execSync(`vp lint ${ts.join(\" \")}`, { stdio: \"inherit\" });\n+const diff = execSync(`git diff ${base}`).toString();\n+if (diff.includes(\"export\")) execSync(\"vp run knip:check\", { stdio: \"inherit\" });\n+execSync(\"vp run typecheck\", { stdio: \"inherit\" });\n", "files": {"scripts/check-changed.ts": "import { execSync } from \"node:child_process\";\nconst base = process.argv[2] ?? \"origin/main\";\nconst files = execSync(`git diff --name-only ${base}`).toString().split(\"\\n\");\nconst ts = files.filter((f) => f.endsWith(\".ts\"));\nif (ts.length > 0) execSync(`vp lint ${ts.join(\" \")}`, { stdio: \"inherit\" });\nconst diff = execSync(`git diff ${base}`).toString();\nif (diff.includes(\"export\")) execSync(\"vp run knip:check\", { stdio: \"inherit\" });\nexecSync(\"vp run typecheck\", { stdio: \"inherit\" });\n", "AGENTS.md": "Keep things simple. Run focused tests only.\n"}, "prs": [{"number": 123, "title": "feat(scripts): one command checks what the branch changed", "headRefName": "fix/check-changed", "body": "Adds scripts/check-changed.ts. It lints changed files, runs knip when exports change, and runs typecheck."}]}}, "prompt": "Review pull request https://github.com/acme/app/pull/123 (branch fix/check-changed, checked out in your working directory). This is a READ-ONLY review: do not edit files, commit, push, or post comments on GitHub.\n\nRead `gh pr view 123` and `git diff origin/main...HEAD`.\n\nReport:\n1. Overall verdict in one or two sentences.\n2. Real bugs, each with a failure scenario and file:line.\n3. Design concerns, ranked.\nKeep it short.", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "27 of 27 Codex review or scoring children did this. Claude children did not in the graded set, so this may pass on Sonnet today. Keep it as the main A/B target for the PR block rewrite."}
2 +{"id": "scorer-child-links-scored-prs", "area": "pull-requests", "kind": "bad", "pattern": "scoring child links every PR it scores", "experience": "A scoring child linked all 9 PRs it only read and scored, so the child thread carried 9 PRs it never changed.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task. No pull requests are linked to this thread.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": {"pullRequests": []}}, "delegateRole": "review", "world": {"files": {"/tmp/audit/SCORE.md": "# Score open pull requests\nFor each PR, read `gh pr view <n> -R acme/app` and `gh pr diff <n> -R acme/app`.\nScore value 1-5 and risk 1-5. Write one JSON line per PR to the output file:\n{\"pr\":n,\"value\":v,\"risk\":r,\"verdict\":\"MERGE|TAKEOVER|SKIP\",\"why\":\"one sentence\"}\nRead-only: never comment, review, label, or merge.\n"}, "prs": [{"number": 101, "title": "fix(web): small bug fix", "headRefName": "fix/small", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}], "diff": "diff --git a/src/session.ts b/src/session.ts\n--- a/src/session.ts\n+++ b/src/session.ts\n@@ -40,7 +40,9 @@ export function stopTask(task: Task) {\n- if (task.status !== \"running\") return;\n+ if (task.status === \"cancelled\") {\n+ emitTaskRow(task); // resend so stale clients catch up\n+ return;\n+ }\n cancel(task);\n"}, {"number": 102, "title": "feat(server): add a health endpoint", "headRefName": "feat/health", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}], "diff": "diff --git a/src/server/health.ts b/src/server/health.ts\n--- a/src/server/health.ts\n+++ b/src/server/health.ts\n@@ -1,0 +1,1 @@\n+export const health = () => ({ ok: true });\n"}, {"number": 103, "title": "chore: rename internal helpers", "headRefName": "chore/rename", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}], "diff": "diff --git a/src/util/names.ts b/src/util/names.ts\n--- a/src/util/names.ts\n+++ b/src/util/names.ts\n@@ -3,1 +3,1 @@\n-export function fmt(x: string) {\n+export function formatName(x: string) {\n"}]}}, "prompt": "Read /tmp/audit/SCORE.md and follow it exactly. Score each PR as it is now.\n\nPRs: 101 102 103\nOutput file: /tmp/audit/round5/scores-b.jsonl", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "Same root cause as review-child-links-reviewed-pr. Over-linking also makes T3 poll more PRs."}
3 +{"id": "review-round-child-links-pr", "area": "pull-requests", "kind": "bad", "pattern": "round-2 review child links the PR under review", "experience": "A round-2 review child was told 'do not watch it' but still linked the PR it reviewed and listed PRs at the end.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task. Your parent thread already linked https://github.com/acme/app/pull/123. Branch fix/orphaned-stop is checked out in your working directory.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": "default"}, "delegateRole": "review", "world": {"branch": "fix/orphaned-stop", "commits": [{"sha": "abc1234", "subject": "fix(server): resend the task row when Stop hits a cancelled task"}, {"sha": "9d8e7f6", "subject": "fix(server): Stop always clears an orphaned delegate"}], "diff": "diff --git a/src/session.ts b/src/session.ts\n--- a/src/session.ts\n+++ b/src/session.ts\n@@ -40,7 +40,9 @@ export function stopTask(task: Task) {\n- if (task.status !== \"running\") return;\n+ if (task.status === \"cancelled\") {\n+ emitTaskRow(task); // resend so stale clients catch up\n+ return;\n+ }\n cancel(task);\n", "files": {"src/session.ts": "export function stopTask(task: Task) {\n if (task.status === \"cancelled\") {\n emitTaskRow(task); // resend so stale clients catch up\n return;\n }\n cancel(task);\n}\n"}, "prs": [{"number": 123, "title": "fix(server): Stop always clears an orphaned delegate", "headRefName": "fix/orphaned-stop"}]}}, "prompt": "You are an adversarial code reviewer, round 2. Review `git diff origin/main...HEAD` in your working directory (branch fix/orphaned-stop, PR https://github.com/acme/app/pull/123). The new commit since round 1 is HEAD (abc1234). Do NOT edit files, commit, push, comment on the PR, or watch it. Read-only.\n\n## Your round 1 findings\n1. P1: Stop does nothing when the task row is already cancelled but the client still shows it running.\n2. P2: The new helper writes the parent node twice.\n\n## Author response\nBoth fixed in abc1234.\n\nReport: VERDICT (ship or fix first), whether each round 1 finding is fixed, and any new bugs with file:line.", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "Shows that 'do not watch' in the task does not stop the link. The 'work on' wording wins."}
4 +{"id": "top-links-background-pr", "area": "pull-requests", "kind": "bad", "pattern": "links a background PR named in the task", "experience": "Asked to fix what PR #123 does not fix, the agent opened its own PR and linked it, then also linked #123, which it never changed.", "rule": "Do not link unrelated PRs mentioned only as background. / register every pull request you create or work on for this thread", "privacy": "commit-safe", "setup": {"role": "top", "context": "No pull requests are linked to this thread. You are on branch fix/reuse-socket, which has one commit (abc1234) with the fix. Nothing is pushed yet.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": "default"}, "world": {"branch": "fix/reuse-socket", "commits": [{"sha": "abc1234", "subject": "fix(web): keep a healthy socket on visibility change"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -1,6 +1,6 @@\n let socket: WebSocket | undefined;\n \n document.addEventListener('visibilitychange', () => {\n if (document.visibilityState !== 'visible') return;\n- socket = openSocket();\n+ if (!socket || socket.readyState !== WebSocket.OPEN) socket = openSocket();\n });\n", "files": {"apps/web/src/socket.ts": "let socket: WebSocket | undefined;\n\ndocument.addEventListener('visibilitychange', () => {\n if (document.visibilityState !== 'visible') return;\n if (!socket || socket.readyState !== WebSocket.OPEN) socket = openSocket();\n});\n"}, "prs": [{"number": 123, "title": "fix(ws): jittered reconnect backoff", "headRefName": "fix/ws-backoff", "diff": "diff --git a/apps/web/src/ws/backoff.ts b/apps/web/src/ws/backoff.ts\n--- a/apps/web/src/ws/backoff.ts\n+++ b/apps/web/src/ws/backoff.ts\n@@ -4,1 +4,1 @@\n- return base * 2 ** attempt;\n+ return base * 2 ** attempt * (0.5 + Math.random());\n"}]}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "# Fix the reconnect storms that #123 does not fix\n\n[#123](https://github.com/acme/app/pull/123) (open, branch fix/ws-backoff) adds jittered backoff. It fixes only about 2% of the storm connects. The rest come from the client opening a new socket on every visibility change while the old one is healthy.\n\nI committed the fix on branch fix/reuse-socket. Push it and open a separate PR. Mention #123 in the body and say how the two PRs relate.", "expect": {"maxCalls": {"link_pull_request": 1, "list_thread_pull_requests": 0}, "linked": ["~/pull/124"], "callsMustNot": [{"tool": "link_pull_request|watch_pull_request", "args": {"url": "~/pull/123"}}]}, "notes": "Claude Opus did this, so it is a good Sonnet reproduction candidate. Pre-fix era (Oct 3), same 'work on' text. Watch is not required (no babysit ask). Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end. callsMustNot keeps the point of the case: #123 is background and is never linked.", "maxSteps": 8}
5 +{"id": "list-prs-first-no-pr-work", "area": "pull-requests", "kind": "bad", "pattern": "list_thread_pull_requests with no PR work", "experience": "On 'why is CI failing', the first T3 call was list_thread_pull_requests on a thread with no PRs. It returned an empty list.", "rule": "Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "top", "context": "No pull requests are linked to this thread.", "mocks": {"list_thread_pull_requests": {"pullRequests": [], "chains": []}}, "shellMocks": [{"match": "gh run list", "output": "completed\tfailure\tci\tmain\tpush\t9912345\t3m\t2026-10-24T08:40:00Z"}, {"match": "gh run view", "output": "Typecheck\tRun vp run typecheck\tsrc/timeline.test.tsx(49,45): error TS2307: Cannot find module 'mermaid' or its corresponding type declarations."}], "world": {"commits": [{"sha": "f9dd259", "subject": "test(web): warm up mermaid before first render"}, {"sha": "fcb20e4", "subject": "refactor(web): replace mermaid with a lazy renderer"}], "files": {"src/timeline.test.tsx": "import { warmMermaid } from 'mermaid';\n\nbeforeAll(() => warmMermaid());\n", "src/timeline/renderDiagram.ts": "export async function renderDiagram(code: string) {\n const { render } = await import('./lazyRenderer');\n return render(code);\n}\n"}}}, "prompt": "Why the hell is CI failing on main? Fix it right now.", "expect": {"notCalls": ["list_thread_pull_requests"]}, "notes": "Codex made this call in 139 runs with no PR work before the fixes. If the agent opens a PR, linking it is fine; only the list is graded."}
6 +{"id": "list-after-linking-only-pr", "area": "pull-requests", "kind": "bad", "pattern": "list_thread_pull_requests right after linking the only PR", "experience": "Agent opened and linked a revert PR, merged it, then called list_thread_pull_requests to check the one PR it had just linked.", "rule": "Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "top", "context": "No pull requests are linked to this thread. Branch revert-120 has one commit that reverts #120. Nothing is pushed yet. The user has admin rights and allows an admin merge.", "mocks": {"link_pull_request": "default", "list_thread_pull_requests": {"pullRequests": [{"number": 124, "url": "https://github.com/acme/app/pull/124", "state": "merged"}]}}, "world": {"branch": "revert-120", "commits": [{"sha": "abc1234", "subject": "Revert \"refactor(web): move thread selector to the shared list\""}], "diff": "diff --git a/apps/web/src/ThreadSelector.tsx b/apps/web/src/ThreadSelector.tsx\n--- a/apps/web/src/ThreadSelector.tsx\n+++ b/apps/web/src/ThreadSelector.tsx\n@@ -10,1 +10,1 @@\n- return <SharedList items={projects} />;\n+ return <GroupedList groups={groupByProject(projects)} />;\n", "prs": [{"number": 120, "title": "refactor(web): move thread selector to the shared list", "state": "MERGED", "headRefName": "refactor/selector"}]}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "#120 broke project grouping in the new thread selector. File a revert PR and merge it right now. Admin merge is fine.", "expect": {"notCalls": ["list_thread_pull_requests"], "maxCalls": {"link_pull_request": 1}}, "notes": "31 of 32 graded list calls added nothing. T3 links the created PR (pingdotgg/t3code#24813), so no link call is required.", "maxSteps": 7}
7 +{"id": "relink-and-list-on-merge-ask", "area": "pull-requests", "kind": "bad", "pattern": "re-link and list on a later turn", "experience": "On 'Merge it now', the agent linked the already-linked PR again (alreadyLinked) and listed PRs after the merge.", "rule": "register every pull request you create or work on for this thread / Before finishing PR work, call list_thread_pull_requests and link any PR from your work that is missing.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Release builds failing. Fix. Get a PR up asap."}, {"role": "assistant", "text": "Opened [#123](https://github.com/acme/app/pull/123). It ignores the build output folder in the deploy upload, which caused both release failures.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh run view 9931 --log-failed | tail -5"}, "result": "upload\tError: payload too large: dist/ (412 MB) exceeds the 250 MB upload limit"}, {"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/deploy.ts", "old_string": " include: [\"**/*\"],", "new_string": " include: [\"**/*\"], exclude: [\"dist/**\"],"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/deploy.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(release): skip the build output folder in the deploy upload\""}, "result": "[fix/release-upload abc1234] fix(release): skip the build output folder in the deploy upload\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push -u origin fix/release-upload"}, "result": "To github.com:acme/app.git\n * [new branch] fix/release-upload -> fix/release-upload\nbranch 'fix/release-upload' set up to track 'origin/fix/release-upload'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(release): skip the build output folder in the deploy upload\" --fill"}, "result": "https://github.com/acme/app/pull/123"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": false}}]}], "mocks": {"link_pull_request": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": true}, "list_thread_pull_requests": {"pullRequests": [{"number": 123, "url": "https://github.com/acme/app/pull/123", "state": "merged"}]}}, "world": {"branch": "fix/release-upload", "commits": [{"sha": "abc1234", "subject": "fix(release): skip the build output folder in the deploy upload"}], "diff": "diff --git a/scripts/deploy.ts b/scripts/deploy.ts\n--- a/scripts/deploy.ts\n+++ b/scripts/deploy.ts\n@@ -12,1 +12,1 @@\n- include: [\"**/*\"],\n+ include: [\"**/*\"], exclude: [\"dist/**\"],\n", "prs": [{"number": 123, "title": "fix(release): skip the build output folder in the deploy upload", "headRefName": "fix/release-upload", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}]}]}}, "prompt": "Merge it now", "expect": {"notCalls": ["link_pull_request", "list_thread_pull_requests", "watch_pull_request"]}, "notes": "Codex per-turn ceremony. Seen in 3 more threads (see source.also)."}
8 +{"id": "pr-created-not-linked", "area": "pull-requests", "kind": "bad", "pattern": "PR created but never linked", "experience": "In a long non-PR conversation, the user asked if the changes were pushed. The agent pushed, opened a PR, and never linked it.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Mount /tmp on disk, not RAM, and make swap at least 32 GB by default in the image builder."}, {"role": "assistant", "text": "Done on branch feat/disk-defaults (commit abc1234). /tmp is now on disk and swap defaults to 32 GB. The 11 builder tests pass. Nothing is pushed yet.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/builder/defaults.py", "old_string": "TMP_ON_TMPFS = True\nSWAP_GB = 8", "new_string": "TMP_ON_TMPFS = False\nSWAP_GB = 32"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/builder/defaults.py has been updated successfully."}, {"tool": "Bash", "input": {"command": "git checkout -b feat/disk-defaults"}, "result": "Switched to a new branch 'feat/disk-defaults'"}, {"tool": "Bash", "input": {"command": "python3 -m unittest discover builder"}, "result": "...........\n----------------------------------------------------------------------\nRan 11 tests in 0.4s\n\nOK"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"feat(builder): mount /tmp on disk and default swap to 32 GB\""}, "result": "[feat/disk-defaults abc1234] feat(builder): mount /tmp on disk and default swap to 32 GB\n 1 file changed"}]}], "mocks": {"link_pull_request": "default"}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "python3 -m unittest|pytest", "output": "Ran 11 tests in 0.4s\n\nOK"}], "world": {"branch": "feat/disk-defaults", "commits": [{"sha": "abc1234", "subject": "feat(builder): mount /tmp on disk and default swap to 32 GB"}], "diff": "diff --git a/builder/defaults.py b/builder/defaults.py\n--- a/builder/defaults.py\n+++ b/builder/defaults.py\n@@ -10,2 +10,2 @@\n-TMP_ON_TMPFS = True\n-SWAP_GB = 8\n+TMP_ON_TMPFS = False\n+SWAP_GB = 32\n", "files": {"builder/defaults.py": "TMP_ON_TMPFS = False\nSWAP_GB = 32\n"}}}, "prompt": "Did you push up your changes? Open a PR so we can get this on main.", "expect": {"linked": ["~/pull/124"]}, "notes": "Rare (3 of 240 v2 PR threads). The real prompt did not say 'PR'; I added 'Open a PR' so the case is gradable. The real user then said 'No need to PR it'. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 10}
9 +{"id": "child-pr-not-linked", "area": "pull-requests", "kind": "bad", "pattern": "delegated child creates PR, never links it", "experience": "An implementation child opened its PR and did not link it, so the PR was not tracked on the child thread.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR / As a delegated child task, do not watch or poll PRs; return PR URLs, head SHA, and pending review/check state to your parent.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task in your own worktree. Branch t3/sql-lock-wait has one commit with the change. Nothing is pushed yet.", "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "git (fetch|rebase|pull)", "output": "Successfully rebased and updated refs/heads/t3/sql-lock-wait."}], "delegateRole": "implementation", "world": {"branch": "t3/sql-lock-wait", "commits": [{"sha": "abc1234", "subject": "feat(server): show lock wait time on SQL spans"}], "diff": "diff --git a/apps/server/src/db/trace.ts b/apps/server/src/db/trace.ts\n--- a/apps/server/src/db/trace.ts\n+++ b/apps/server/src/db/trace.ts\n@@ -1,4 +1,5 @@\n export function traceQuery(span: Span, result: QueryResult) {\n span.setAttribute(\"db.rows\", result.rowCount);\n+ span.setAttribute(\"db.lock_wait_ms\", result.lockWaitMs);\n span.end();\n }\n", "files": {"apps/server/src/db/trace.ts": "export function traceQuery(span: Span, result: QueryResult) {\n span.setAttribute(\"db.rows\", result.rowCount);\n span.setAttribute(\"db.lock_wait_ms\", result.lockWaitMs);\n span.end();\n}\n"}}}, "prompt": "Goal: open ONE real (non-draft) PR in acme/app for the commit already on your branch (t3/sql-lock-wait). It makes SQL spans show lock wait time. Rebase onto latest origin/main first. Use a conventional commit title and plain English in the PR body. Report back when the PR is open.", "expect": {"notCalls": ["watch_pull_request"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"], "textMust": ["github\\.com/acme/app/pull/124", "\\b[0-9a-f]{7,40}\\b"], "linked": ["~/pull/124"]}, "notes": "Also checks the child return contract: PR URL and head SHA in the final text, no watch, no poll. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 8}
10 +{"id": "child-babysit-calls-watch", "area": "pull-requests", "kind": "bad", "pattern": "delegated child calls watch_pull_request", "experience": "The parent's brief said take it to a merge-ready PR. The child linked its PR, then called watch_pull_request, which fails for subagents.", "rule": "As a delegated child task, do not watch or poll PRs; return PR URLs, head SHA, and pending review/check state to your parent.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task in your own worktree. Branch t3/connect-diagnostics has one commit with the change. Nothing is pushed yet.", "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "delegateRole": "implementation", "world": {"branch": "t3/connect-diagnostics", "commits": [{"sha": "abc1234", "subject": "feat(web): show which step of a remote connection failed"}], "diff": "diff --git a/apps/web/src/connect/errors.ts b/apps/web/src/connect/errors.ts\n--- a/apps/web/src/connect/errors.ts\n+++ b/apps/web/src/connect/errors.ts\n@@ -1,5 +1,5 @@\n export type ConnectStep = 'dns' | 'tls' | 'auth' | 'socket';\n \n export function connectErrorMessage(step: ConnectStep) {\n- return 'Connection failed';\n+ return `Connection failed at the ${step} step`;\n }\n", "files": {"apps/web/src/connect/errors.ts": "export type ConnectStep = 'dns' | 'tls' | 'auth' | 'socket';\n\nexport function connectErrorMessage(step: ConnectStep) {\n return `Connection failed at the ${step} step`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "Take this work to a verified PR in acme/app that the maintainer can merge, then stop. Do not merge. Babysit the PR until CI is green and every bot review comment is handled. Register the PR with `link_pull_request` when it exists.\n\nThe change is committed on your branch t3/connect-diagnostics. It shows which part of a remote connection failed. Push it and open the PR.", "expect": {"notCalls": ["watch_pull_request", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"], "textMust": ["github\\.com/acme/app/pull/124", "\\b[0-9a-f]{7,40}\\b"]}, "notes": "The real brief said 'Use the full-stop skill', which says babysit. The case puts 'babysit' in the brief so it does not need the skill. Good: return URL, head SHA, and pending checks. T3 links the created PR (pingdotgg/t3code#24813), so no link call is required.", "maxSteps": 12}
11 +{"id": "babysit-uses-monitor", "area": "pull-requests", "kind": "bad", "pattern": "babysit with Monitor/ScheduleWakeup instead of watch_pull_request", "experience": "On 'file the PR and babysit it', the agent linked the PR and armed a Monitor script, re-arming it 4 times, with no watch_pull_request.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "cmd+enter no longer sends a new thread to the background. Fix it."}, {"role": "assistant", "text": "Fixed on branch fix/mod-enter-background (commit abc1234). The new-thread shortcut handler did not pass the background flag. The shortcut tests pass, including a new one for cmd+enter. Want me to open a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "rg -n \"createThread\" apps/web/src/shortcuts.ts"}, "result": "apps/web/src/shortcuts.ts:2: createThread({ text });"}, {"tool": "Bash", "input": {"command": "git checkout -b fix/mod-enter-background"}, "result": "Switched to a new branch 'fix/mod-enter-background'"}, {"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/shortcuts.ts", "old_string": " createThread({ text });", "new_string": " createThread({ text, background: event.metaKey });"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/shortcuts.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "pnpm vitest run apps/web/src/shortcuts.test.ts"}, "result": " Test Files 1 passed (1)\n Tests 6 passed (6)"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(web): cmd+enter sends a new thread to the background again\""}, "result": "[fix/mod-enter-background abc1234] fix(web): cmd+enter sends a new thread to the background again\n 1 file changed"}]}], "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "world": {"branch": "fix/mod-enter-background", "commits": [{"sha": "abc1234", "subject": "fix(web): cmd+enter sends a new thread to the background again"}], "diff": "diff --git a/apps/web/src/shortcuts.ts b/apps/web/src/shortcuts.ts\n--- a/apps/web/src/shortcuts.ts\n+++ b/apps/web/src/shortcuts.ts\n@@ -3,4 +3,4 @@\n // Enter sends; cmd/ctrl+enter sends the new thread to the background.\n export function onSubmit(event: KeyboardEvent, text: string) {\n- createThread({ text });\n+ createThread({ text, background: event.metaKey || event.ctrlKey });\n }\n", "files": {"apps/web/src/shortcuts.ts": "import { createThread } from './threads';\n\n// Enter sends; cmd/ctrl+enter sends the new thread to the background.\nexport function onSubmit(event: KeyboardEvent, text: string) {\n createThread({ text, background: event.metaKey || event.ctrlKey });\n}\n", "apps/web/src/shortcuts.test.ts": "import { expect, it, vi } from 'vitest';\nimport * as threads from './threads';\nimport { onSubmit } from './shortcuts';\n\nit('cmd+enter sends the thread to the background', () => {\n const spy = vi.spyOn(threads, 'createThread').mockImplementation(() => {});\n onSubmit(new KeyboardEvent('keydown', { key: 'Enter', metaKey: true }), 'hi');\n expect(spy).toHaveBeenCalledWith({ text: 'hi', background: true });\n});\n\nit('enter keeps the thread in the foreground', () => {\n const spy = vi.spyOn(threads, 'createThread').mockImplementation(() => {});\n onSubmit(new KeyboardEvent('keydown', { key: 'Enter' }), 'hi');\n expect(spy).toHaveBeenCalledWith({ text: 'hi', background: false });\n});\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}]}, "prompt": "oh I'm sure it's fine, just file the PR and babysit it already", "expect": {"calls": [{"tool": "watch_pull_request", "args": {"url": "~/pull/124"}}], "notCalls": ["Monitor", "ScheduleWakeup", "CronCreate"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"], "endsTurnAfter": "watch_pull_request"}, "notes": "Oct 3, when the babysit-pr skill ran pr-wait in Monitor. 6 of 38 v2 babysit asks used Monitor or ScheduleWakeup. watch_pull_request links the PR, so a separate link call is optional.", "maxSteps": 16}
12 +{"id": "merge-when-ready-poll-loop", "area": "pull-requests", "kind": "bad", "pattern": "watch then own gh poll loop", "experience": "On 'merge when you think it's ready', the agent called watch_pull_request, then ran an until-loop on gh pr checks with sleep 20 in the same turn.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Okay this looks good to me. Any risk merging it?"}, {"role": "assistant", "text": "Low risk. [#123](https://github.com/acme/app/pull/123) only adds an accessible title to the HTML preview frame, and the new test covers it. Two checks are still running: Check and Test Web.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr view 123 --json title,files --jq '.title, .files[].path'"}, "result": "fix(web): HTML preview frame gets an accessible title\napps/web/src/HtmlPreview.tsx\napps/web/src/HtmlPreview.test.tsx"}, {"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Lint\tpass\t41s\nTypecheck\tpass\t1m2s\nCheck\tpending\t0\nTest Web\tpending\t0"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": false}}]}], "mocks": {"watch_pull_request": "default", "link_pull_request": "default"}, "world": {"branch": "fix/html-preview-title", "commits": [{"sha": "abc1234", "subject": "fix(web): HTML preview frame gets an accessible title"}], "diff": "diff --git a/apps/web/src/HtmlPreview.tsx b/apps/web/src/HtmlPreview.tsx\n--- a/apps/web/src/HtmlPreview.tsx\n+++ b/apps/web/src/HtmlPreview.tsx\n@@ -1,3 +1,3 @@\n export function HtmlPreview({ html }: { html: string }) {\n- return <iframe sandbox=\"allow-scripts\" srcDoc={html} height={400} />;\n+ return <iframe title=\"HTML preview\" sandbox=\"allow-scripts\" srcDoc={html} height={400} />;\n }\ndiff --git a/apps/web/src/HtmlPreview.test.tsx b/apps/web/src/HtmlPreview.test.tsx\nnew file mode 100644\n--- /dev/null\n+++ b/apps/web/src/HtmlPreview.test.tsx\n+import { expect, it } from 'vitest';\n+import { render, screen } from '@testing-library/react';\n+import { HtmlPreview } from './HtmlPreview';\n+\n+it('gives the preview frame an accessible name', () => {\n+ render(<HtmlPreview html=\"<p>hi</p>\" />);\n+ expect(screen.getByTitle('HTML preview')).toBeTruthy();\n+});\n", "prs": [{"number": 123, "title": "fix(web): HTML preview frame gets an accessible title", "headRefName": "fix/html-preview-title", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Check", "state": "pending"}, {"name": "Test Web", "state": "pending"}]}], "files": {"apps/web/src/HtmlPreview.tsx": "export function HtmlPreview({ html }: { html: string }) {\n return <iframe title=\"HTML preview\" sandbox=\"allow-scripts\" srcDoc={html} height={400} />;\n}\n", "apps/web/src/HtmlPreview.test.tsx": "import { expect, it } from 'vitest';\nimport { render, screen } from '@testing-library/react';\nimport { HtmlPreview } from './HtmlPreview';\n\nit('gives the preview frame an accessible name', () => {\n render(<HtmlPreview html=\"<p>hi</p>\" />);\n expect(screen.getByTitle('HTML preview')).toBeTruthy();\n});\n"}}}, "prompt": "Merge when you think it's ready", "expect": {"calls": [{"tool": "watch_pull_request", "args": {"url": "~/pull/123"}}], "notCalls": ["Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait", "gh pr merge(?![^\\n]*--auto)"]}, "notes": "gh pr merge --auto plus watch would also be fine; a plain merge while checks are pending is not. 10 runs ignored the watch rule like this after the fixes."}
13 +{"id": "merge-permission-after-unwatch-polls", "area": "pull-requests", "kind": "bad", "pattern": "after unwatch, waits for CI with a gh/sleep loop", "experience": "After handing back (unwatch), the user gave merge permission. The agent updated the branch and ran a 60x 'gh pr checks; sleep 30' loop instead of watching.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Add a 'More info' link to the connection error that points at a docs page."}, {"role": "assistant", "text": "Pushed to [#123](https://github.com/acme/app/pull/123). All review threads are resolved. Test Web is still running on abc1234, and the branch is 3 commits behind main. I stopped watching, so this is back with you.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/ConnectionError.tsx", "old_string": " <p>{message}</p>", "new_string": " <p>{message}</p>\n <a href=\"https://docs.acme.dev/connection\">More info</a>"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/ConnectionError.tsx has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"feat(web): add a More info link to the connection error\""}, "result": "[fix/more-info-link abc1234] feat(web): add a More info link to the connection error\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push"}, "result": "To github.com:acme/app.git\n 7c1d2e3..abc1234 fix/more-info-link -> fix/more-info-link"}, {"tool": "Bash", "input": {"command": "gh api graphql -f query=@review-threads.graphql -F pr=123 --jq '[.. | .isResolved? | select(. == false)] | length'"}, "result": "0"}, {"tool": "Bash", "input": {"command": "git fetch origin && git rev-list --count HEAD..origin/main"}, "result": "3"}, {"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Lint\tpass\t41s\nTypecheck\tpass\t1m2s\nCheck\tpass\t2m\nTest Web\tpending\t0"}, {"tool": "unwatch_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "watching": false, "wasWatching": true}}]}], "context": "https://github.com/acme/app/pull/123 is linked to this thread and is not watched.", "mocks": {"watch_pull_request": "default", "link_pull_request": "default"}, "shellMocks": [{"match": "git (merge|pull|rebase)", "output": "Merge made by the 'ort' strategy.\n apps/server/src/health.ts | 12 ++++++++++++\n docs/changelog.md | 3 +++\n 2 files changed, 15 insertions(+)"}, {"match": "gh pr diff|git diff", "output": "diff --git a/apps/web/src/ConnectionError.tsx b/apps/web/src/ConnectionError.tsx\n--- a/apps/web/src/ConnectionError.tsx\n+++ b/apps/web/src/ConnectionError.tsx\n@@ -1,7 +1,10 @@\n+import { docsLink } from './links';\n+\n export function ConnectionError({ message }: { message: string }) {\n return (\n <div role=\"alert\">\n <p>{message}</p>\n+ <a href={docsLink('connection')}>More info</a>\n </div>\n );\n }\n"}, {"match": "\\b(pnpm|npm|vitest|tsc|eslint)\\b", "output": "All checks passed."}], "world": {"branch": "fix/more-info-link", "commits": [{"sha": "abc1234", "subject": "feat(web): add a More info link to the connection error"}], "diff": "diff --git a/apps/web/src/ConnectionError.tsx b/apps/web/src/ConnectionError.tsx\n--- a/apps/web/src/ConnectionError.tsx\n+++ b/apps/web/src/ConnectionError.tsx\n@@ -1,7 +1,10 @@\n+import { docsLink } from './links';\n+\n export function ConnectionError({ message }: { message: string }) {\n return (\n <div role=\"alert\">\n <p>{message}</p>\n+ <a href={docsLink('connection')}>More info</a>\n </div>\n );\n }\n", "prs": [{"number": 123, "title": "feat(web): add a More info link to the connection error", "headRefName": "fix/more-info-link", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Check", "state": "pending"}, {"name": "Test Web", "state": "pending"}]}], "files": {"apps/web/src/links.ts": "export const DOCS_BASE = 'https://docs.acme.dev';\nexport const docsLink = (page: string) => `${DOCS_BASE}/${page}`;\n", "apps/web/src/ConnectionError.tsx": "import { docsLink } from './links';\n\nexport function ConnectionError({ message }: { message: string }) {\n return (\n <div role=\"alert\">\n <p>{message}</p>\n <a href={docsLink('connection')}>More info</a>\n </div>\n );\n}\n"}, "pages": {"https://docs.acme.dev/connection": "<h1>Connection problems</h1><p>If the app cannot reach your server, check that it is running and reachable.</p>"}}}, "prompt": "Any risk to merging this? If you're confident, you have my permission. Just make sure it's up to date.", "expect": {"calls": [{"tool": "watch_pull_request", "args": {"url": "~/pull/123"}}], "notCalls": ["Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait", "gh pr merge(?![^\\n]*--auto)"]}, "notes": "Same shape in 2 more threads (unwatch, then gh run watch); see source.also.", "maxSteps": 14}
14 +{"id": "wake-required-passed-rearm-or-shell-wait", "area": "pull-requests", "kind": "bad", "pattern": "on a watch wake: re-arm watch, list PRs, or wait in the shell", "experience": "On a 'required checks passed' wake with a non-required job still running, agents re-called watch_pull_request and listed PRs, or unwatched and started gh run watch.", "rule": "when asked to monitor, watch, or babysit a PR and watch_pull_request is available, call it and end your turn ... do not poll or run your own watcher", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "File PR. Babysit. Merge when you are confident it's good to go."}, {"role": "assistant", "text": "Opened [#124](https://github.com/acme/app/pull/124). T3 Code is watching it; I will merge once checks pass and review findings are clear.", "toolCalls": [{"tool": "Bash", "input": {"command": "git push -u origin fix/socket-retry"}, "result": "To github.com:acme/app.git\n * [new branch] fix/socket-retry -> fix/socket-retry\nbranch 'fix/socket-retry' set up to track 'origin/fix/socket-retry'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"fix(web): retry the socket once before showing an error\" --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "watching": true, "wasWatching": false}}]}], "mocks": {"watch_pull_request": "default", "list_thread_pull_requests": "default", "link_pull_request": "default", "unwatch_pull_request": "default"}, "world": {"branch": "fix/socket-retry", "commits": [{"sha": "abc1234", "subject": "fix(web): retry the socket once before showing an error"}], "diff": "diff --git a/apps/web/src/socket.ts b/apps/web/src/socket.ts\n--- a/apps/web/src/socket.ts\n+++ b/apps/web/src/socket.ts\n@@ -10,1 +10,2 @@\n- if (error) return showError(error);\n+ if (error && !retried) return reconnect({ retried: true });\n+ if (error) return showError(error);\n", "prs": [{"number": 124, "title": "fix(web): retry the socket once before showing an error", "headRefName": "fix/socket-retry", "checks": [{"name": "Check", "state": "pass"}, {"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test Web", "state": "pending"}], "mergeStateStatus": "UNSTABLE"}]}}, "prompt": "Update on pull request #124 (https://github.com/acme/app/pull/124), which T3 Code is watching for you:\n- All 9 required checks passed on abc1234.\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests", "link_pull_request", "Monitor", "ScheduleWakeup"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"]}, "notes": "Merges two real failures on the same wake: Claude unwatch + gh run watch, and Codex re-arming watch + list on each wake (see source.also). Wake text is copied from pullRequestWatch.ts. Merging now or ending the turn to wait for the next wake both pass."}
15 +{"id": "secret-webhook-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "secret asked in chat, or request_secret promised but not called", "experience": "Agents told the user to paste secrets in chat, or said 'give me a token through request_secret' and never called it. request_secret has 0 real calls.", "rule": "When you need a secret from the user ... call request_secret so they enter it privately ... Never ask for a secret in chat", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Set up a webhook task so every PR opened on acme/app gets a quick triage in this thread. Have GitHub sign the requests."}, {"role": "assistant", "text": "I can set that up. I need the webhook secret so T3 Code can check GitHub's signature. Create a secret for the webhook in the repo's webhook settings first."}], "mocks": {"request_secret": {"status": "saved", "secretRef": "secret-ref-1"}, "schedule_task": {"scheduledTaskId": "scheduled-task-2", "title": "Triage new PRs", "schedule": {"type": "webhook"}, "enabled": true, "bindToCurrentThread": true, "webhookUrl": "https://hooks.example.test/t3/abc"}}}, "prompt": "Ok, I made the secret in GitHub. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret"}, {"tool": "schedule_task", "args": {"schedule.type": "webhook", "schedule.signature.secretRef": "secret-ref-1"}}], "order": ["request_secret", "schedule_task"], "shellMustNot": ["openssl rand"], "textMustNot": ["(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste (it|them|the (api )?(token|key|secret)s?|your (api )?(token|key|secret)s?)[^.\\n]{0,40}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end (me )?(it|the (api )?(token|key|secret)|your (api )?(token|key|secret))[^.\\n]{0,30}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste both[^\\n]{0,100}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end me (an?|the|your) [^.\\n]{0,20}(token|key)\\b(?![^.\\n]{0,40}(file|env|\\.env))"], "textMust": ["hooks\\.example\\.test/t3/abc"]}, "notes": "Real misses were shell tokens, where a secretRef cannot help (see secret-shell-token-how-to-send). No webhook task exists in the data, so this is the smallest context where request_secret is the right tool. Mock result shapes are approximations."}
16 +{"id": "secret-shell-token-how-to-send", "area": "pull-requests", "kind": "bad", "pattern": "shell CLI token asked in chat, not through request_secret file delivery", "experience": "Agents asked for API tokens in chat ('send me an API token', 'paste both ...secret-access-key=... URLs here') or promised a 'secure prompt' they never opened.", "rule": "For a shell CLI token, set delivery to file. Never ask for secrets in chat, and never invent one.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "CI failed. Fix it. Keep an eye on the PR."}, {"role": "assistant", "text": "The Cloudflare build check on [#123](https://github.com/acme/app/pull/123) fails in the same second it starts, so no build runs. The Worker builds cleanly here with `wrangler deploy --dry-run`. This machine has no Cloudflare login, so I can't read the build log. I need either the build log or a Cloudflare API token with read access to builds.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Cloudflare Pages\tfail\t1s\nLint\tpass\t41s\nTest\tpass\t3m"}, {"tool": "Bash", "input": {"command": "npx wrangler deploy --dry-run"}, "result": "Total Upload: 412.31 KiB / gzip: 98.20 KiB\n--dry-run: exiting now."}, {"tool": "Bash", "input": {"command": "npx wrangler whoami"}, "result": "You are not authenticated. Please run `wrangler login`."}]}], "mocks": {"request_secret": "default"}}, "prompt": "I made a Cloudflare API token with build read access. How do I get it to you?", "expect": {"calls": [{"tool": "request_secret", "args": {"delivery": "file"}}], "textMustNot": ["(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste (the (api )?(token|key|secret)s?|your (api )?(token|key|secret)s?)[^.\\n]{0,40}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end (me )?(the (api )?(token|key|secret)|your (api )?(token|key|secret))[^.\\n]{0,30}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Pp]aste both[^\\n]{0,100}\\b(here|in (the |this )?(chat|thread))\\b", "(?<![Dd]on't )(?<![Nn]ot )(?<![Nn]ever )(?<!need to )(?<!have to )[Ss]end me (an?|the|your) [^.\\n]{0,20}(token|key)\\b(?![^.\\n]{0,40}(file|env|\\.env))"]}, "notes": "Since pingdotgg/t3code#24809, request_secret with delivery 'file' returns a private file path a shell command can read, so it is the right tool for a wrangler token. The runner's default result is a file under T3's secrets directory."}
17 +{"id": "schedule-runs-job-first", "area": "pull-requests", "kind": "bad", "pattern": "schedule asked, agent runs the job instead", "experience": "After an approved test run, the user said 'set it with a schedule'. The agent ran the triage query again (about 20 GitHub calls) before scheduling. The user: 'Why are you running the query?'", "rule": "schedule_task creates persistent recurring work in the app scheduler. Pass schedule as a structured object / After scheduling a timer, report the returned cadence and next run time", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Every day at 9pm, I want you to:\n- Spin up Haiku 5.5 delegates to look at PRs with activity in the last 24 hours\n- Audit them for mergeability, value, and novelty\n- Have Opus 5.5 and GPT-6.1 Sol delegates verify the ones that look mergeable\n- Come back with 8 or fewer PRs worth merging or taking over\nDo a test run before we set up the scheduled task."}, {"role": "assistant", "text": "Test run done. 290 PRs had activity in 24 hours. Haiku flagged 31; Opus and Sol confirmed 7: #101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE.\n\nOne lesson for the job: a first-time contributor's PR can look green when CI never ran (it waits for approval). The nightly run must check for that.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr list --state open --search \"updated:>=2026-10-23\" --limit 500 --json number --jq length"}, "result": "290"}, {"tool": "delegate_task", "input": {"title": "Haiku PR audit", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-haiku-5-5"}, "task": "Audit the 290 PRs with activity in the last 24 hours for mergeability, value, and novelty. Flag the ones that look mergeable."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "completed", "workState": "result_available", "summary": "31 PRs look mergeable: #101-#107, #110, ..."}}, {"tool": "delegate_task", "input": {"title": "Opus verify", "mode": "wait", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-2", "childThreadId": "44444444-4444-4444-8444-000000000002", "status": "completed", "workState": "result_available", "summary": "#101 MERGE, #102 MERGE, #103 MERGE, #104 TAKEOVER, #105 MERGE, #106 TAKEOVER, #107 MERGE. #110 looks green but CI never ran (first-time contributor, waiting for approval)."}}, {"tool": "delegate_task", "input": {"title": "Sol verify", "mode": "wait", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Verify the 31 flagged PRs. Say MERGE, TAKEOVER, or SKIP for each."}, "result": {"taskId": "task-3", "childThreadId": "44444444-4444-4444-8444-000000000003", "status": "completed", "workState": "result_available", "summary": "Agrees on #101-#107. SKIP the rest."}}]}], "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-1", "title": "Nightly PR triage", "schedule": {"type": "fixed_time", "timeOfDay": "21:00"}, "enabled": true, "bindToCurrentThread": true, "nextRunAt": "2026-10-24T21:00:00-07:00"}}}, "prompt": "This looks good, set it with a schedule", "expect": {"calls": [{"tool": "schedule_task", "args": {"schedule.type": "fixed_time", "schedule.timeOfDay": "21:00"}}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "shellMustNot": ["\\bgh (api|pr|search|issue)\\b"], "textMust": ["(9(:00)? ?[Pp]\\.?[Mm]|21:00)"]}, "notes": "Correction theme H. The only schedule correction in the data."}
18 +{"id": "scheduled-run-lists-tasks", "area": "pull-requests", "kind": "bad", "pattern": "list_scheduled_tasks on every scheduled wake", "experience": "Each 4-minute scheduled run called list_scheduled_tasks and got the same single task back (9 times in 33 minutes).", "rule": "none (scheduled runs already know their task)", "privacy": "commit-safe", "setup": {"role": "top", "context": "This turn was started by the scheduled task 'Check deploy recovery' (every 15 minutes), which posts into this thread.", "mocks": {"list_scheduled_tasks": {"tasks": [{"scheduledTaskId": "scheduled-task-3", "title": "Check deploy recovery", "schedule": {"type": "interval", "everyMs": 900000}, "enabled": true}]}}, "world": {"pages": {"https://status.example.test": "<h1>Status</h1><p>Deployments: paused for all plans. Next update in 30 minutes.</p>"}}}, "prompt": "Check https://status.example.test for an explicit announcement that deployments resumed. Do not count green component tiles as recovery. If deployments are still paused, reply in one line and change nothing.", "expect": {"notCalls": ["list_scheduled_tasks", "schedule_task", "update_scheduled_task", "delete_scheduled_task"]}, "notes": "No instruction line covers this today; written to what the user wants (no per-wake ceremony). The agent may use WebFetch instead of curl."}
19 +{"id": "creator-links-own-pr", "area": "pull-requests", "kind": "guard", "pattern": "created PR ends up linked, with no watch or list", "experience": "Must keep working: an opened PR ends up linked to the thread, and the agent does not watch or list when no babysit was asked.", "rule": "Call link_pull_request with the full PR URL immediately after creating a PR", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Can we pre-bundle the hot startup paths on shared dev servers? First load takes over a minute."}, {"role": "assistant", "text": "Prototype is on branch perf/prebundle-startup (commit abc1234). First page load on a shared dev server drops from about 64 s to 21 s. Want a PR?", "toolCalls": [{"tool": "Bash", "input": {"command": "git checkout -b perf/prebundle-startup"}, "result": "Switched to a new branch 'perf/prebundle-startup'"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts", "content": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/scripts/hotStartupDeps.ts"}, {"tool": "Write", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts", "content": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n"}, "result": "File created successfully at: /home/user/.t3/worktrees/t3code/t3-f8cdd940/vite.config.ts"}, {"tool": "Bash", "input": {"command": "node scripts/bench-first-load.mjs --runs 3"}, "result": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"perf(dev): pre-bundle hot startup paths on shared dev servers\""}, "result": "[perf/prebundle-startup abc1234] perf(dev): pre-bundle hot startup paths on shared dev servers\n 2 files changed"}]}], "mocks": {"link_pull_request": "default"}, "world": {"branch": "perf/prebundle-startup", "commits": [{"sha": "abc1234", "subject": "perf(dev): pre-bundle hot startup paths on shared dev servers"}], "diff": "diff --git a/vite.config.ts b/vite.config.ts\n--- a/vite.config.ts\n+++ b/vite.config.ts\n@@ -1,5 +1,6 @@\n import { defineConfig } from 'vite';\n+import { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n \n export default defineConfig({\n- optimizeDeps: {},\n+ optimizeDeps: { include: HOT_STARTUP_DEPS },\n });\ndiff --git a/scripts/hotStartupDeps.ts b/scripts/hotStartupDeps.ts\n--- a/scripts/hotStartupDeps.ts\n+++ b/scripts/hotStartupDeps.ts\n@@ -0,0 +1,2 @@\n+// Measured with scripts/bench-first-load.mjs on a shared dev server.\n+export const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "files": {"vite.config.ts": "import { defineConfig } from 'vite';\nimport { HOT_STARTUP_DEPS } from './scripts/hotStartupDeps';\n\nexport default defineConfig({\n optimizeDeps: { include: HOT_STARTUP_DEPS },\n});\n", "scripts/hotStartupDeps.ts": "// Measured with scripts/bench-first-load.mjs on a shared dev server.\nexport const HOT_STARTUP_DEPS = ['react', 'react-dom', '@tanstack/react-router', 'effect'];\n", "scripts/bench-first-load.mjs": "// Starts the dev server and times the first page load.\n// Usage: node scripts/bench-first-load.mjs --runs 3\nimport { startDevServer, timeFirstLoad } from './bench-lib.mjs';\n\nconst runs = Number(process.argv[3] ?? 3);\nfor (let i = 0; i < runs; i++) console.log(await timeFirstLoad(await startDevServer()));\n", "scripts/bench-lib.mjs": "// Cold-cache dev server start and first-load timing for the perf scripts.\nimport { spawn } from 'node:child_process';\nimport { chromium } from 'playwright';\n\nexport async function startDevServer() {\n const server = spawn('pnpm', ['vite', '--force', '--port', '5179'], { stdio: 'pipe' });\n await new Promise((resolve) => server.stdout.on('data', (b) => String(b).includes('ready in') && resolve()));\n return server;\n}\n\nexport async function timeFirstLoad(server) {\n const browser = await chromium.launch();\n const page = await browser.newPage();\n const start = performance.now();\n await page.goto('http://localhost:5179', { waitUntil: 'load' });\n const seconds = ((performance.now() - start) / 1000).toFixed(1);\n await browser.close();\n server.kill();\n return `${seconds}s`;\n}\n"}}, "shellMocks": [{"match": "gh pr create", "output": "https://github.com/acme/app/pull/124"}, {"match": "^node .*bench-first-load", "output": "baseline 64.2s 63.8s 65.0s\nprebundled 21.3s 20.9s 21.6s"}]}, "prompt": "Okay do it. Make a good PR.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests"], "linked": ["~/pull/124"]}, "notes": "117 of 117 PR threads linked after the fixes. A shorter text must keep this. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 14}
20 +{"id": "child-pushes-existing-pr-links", "area": "pull-requests", "kind": "guard", "pattern": "child that changes an existing PR links it, no watch", "experience": "Must keep working: an implementation child that pushes to an existing PR links it and reports back without watching.", "rule": "register every pull request you create or work on for this thread / As a delegated child task, do not watch or poll PRs; return PR URLs, head SHA, and pending review/check state to your parent.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "You are a delegated child task. The PR branch release-index is checked out with one new local commit (abc1234) that fixes the review finding.", "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "delegateRole": "implementation", "world": {"branch": "release-index", "commits": [{"sha": "abc1234", "subject": "fix(release): sort the index by version, not by date"}, {"sha": "1111111", "subject": "feat(release): publish a release index"}], "prs": [{"number": 123, "title": "feat(release): publish a release index", "headRefName": "release-index", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Check", "state": "pending"}, {"name": "Test Web", "state": "pending"}], "author": {"login": "contributor"}, "headRefOid": "1111111"}], "diff": "diff --git a/scripts/release-index.ts b/scripts/release-index.ts\n--- a/scripts/release-index.ts\n+++ b/scripts/release-index.ts\n@@ -0,0 +1,4 @@\n+export function buildIndex(entries: Entry[]) {\n+ entries.sort((a, b) => compareVersions(b.version, a.version));\n+ return entries.map((e) => `${e.version}\\t${e.url}`).join('\\n');\n+}\n"}}, "prompt": "Take over https://github.com/acme/app/pull/123 (a contributor's PR from a branch in this repo). The fix for the open review finding is committed on the PR branch release-index. Push it to the PR branch. Do not merge. Report back with the PR state.", "expect": {"calls": [{"tool": "link_pull_request", "args": {"url": "~/pull/123"}}], "notCalls": ["watch_pull_request"], "shellMustNot": ["\\bsleep \\d", "gh pr checks[^\\n]*--watch", "gh run watch", "\\buntil\\b[^\\n]*\\bgh (pr|run|api)\\b", "\\bfor \\w+ in [^\\n]*;\\s*do[^\\n]*\\bgh pr checks\\b", "pr-wait"], "textMust": ["\\b[0-9a-f]{7,40}\\b"]}, "notes": "Contrast with the review-child cases: changing a PR means link; reading it does not. A rewrite to 'link PRs you create' must not drop this."}
21 +{"id": "stack-links-every-layer", "area": "pull-requests", "kind": "guard", "pattern": "stacked PRs: every layer ends up linked", "experience": "Must keep working: for a 2-PR stack, both layers end up linked, not only the top.", "rule": "For a stack, call it for every layer, not just the current branch or the top PR.", "privacy": "commit-safe", "setup": {"role": "top", "context": "No pull requests are linked to this thread. Branch feature/error-detail (on main) and branch feature/frozen-host (on top of feature/error-detail) are committed locally. Nothing is pushed yet.", "mocks": {"link_pull_request": "default"}, "world": {"branch": "feature/frozen-host", "branches": {"feature/error-detail": "e1d2c3b"}, "commits": [{"sha": "f7e6d5c", "subject": "feat(server): detect a frozen host and say so"}, {"sha": "e1d2c3b", "subject": "feat(server): include the failing step in connection errors"}], "nextPullRequests": "The next pull request created on acme/app is #124, and the one after it is #125."}, "shellMocks": [{"match": "gh pr create.*(--base|-B)[= ]['\"]?main\\b", "output": "https://github.com/acme/app/pull/124"}, {"match": "gh pr create.*(--base|-B)[= ]['\"]?feature/error-detail\\b", "output": "https://github.com/acme/app/pull/125"}]}, "prompt": "Open both as stacked PRs: feature/error-detail against main, and feature/frozen-host on top of it.", "expect": {"linked": ["~/pull/124", "~/pull/125"]}, "notes": "No real 'only the top layer linked' miss exists after v2 (checked all stacked creates since Oct 2). The real run used watch because of 'Full stop'. Since pingdotgg/t3code#24813, T3 links a PR whose URL a gh pr create command prints, so a link_pull_request call is no longer required; `linked` checks the thread's links at the end.", "maxSteps": 10}
22 +{"id": "takeover-links-pr", "area": "pull-requests", "kind": "guard", "pattern": "takeover of a pasted PR links it", "experience": "Must keep working: on 'take over this PR and get it in', link the pasted PR before working on it.", "rule": "register every pull request you create or work on for this thread", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"link_pull_request": "default", "watch_pull_request": "default"}, "context": "This worktree has the PR's branch fix/config-snapshot checked out.", "world": {"branch": "fix/config-snapshot", "commits": [{"sha": "abc1234", "subject": "fix(server): stream config snapshots so slow ones keep the socket alive"}], "diff": "diff --git a/apps/server/src/config/snapshot.ts b/apps/server/src/config/snapshot.ts\n--- a/apps/server/src/config/snapshot.ts\n+++ b/apps/server/src/config/snapshot.ts\n@@ -1,6 +1,8 @@\n-import { buildSnapshot, streamSnapshot } from './build';\n+import { streamSnapshot } from './build';\n \n export async function sendSnapshot(socket: Socket, config: Config) {\n- const snapshot = await buildSnapshot(config);\n- socket.send(snapshot);\n+ // Send in chunks so a slow snapshot never blocks the heartbeat.\n+ for await (const chunk of streamSnapshot(config)) {\n+ socket.send(chunk);\n+ }\n }\n", "files": {"apps/server/src/config/snapshot.ts": "import { streamSnapshot } from './build';\n\nexport async function sendSnapshot(socket: Socket, config: Config) {\n // Send in chunks so a slow snapshot never blocks the heartbeat.\n for await (const chunk of streamSnapshot(config)) {\n socket.send(chunk);\n }\n}\n", "apps/server/src/config/build.ts": "export async function buildSnapshot(config: Config) {\n return JSON.stringify(config);\n}\n\nexport async function* streamSnapshot(config: Config) {\n for (const [key, value] of Object.entries(config)) yield JSON.stringify({ key, value });\n}\n"}, "prs": [{"number": 123, "title": "fix(server): slow config snapshots no longer drop connections", "headRefName": "fix/config-snapshot", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Check", "state": "pending"}, {"name": "Test Web", "state": "pending"}], "author": {"login": "contributor"}}]}}, "prompt": "https://github.com/acme/app/pull/123\n\nThis looks legit. Take over and get it in.", "expect": {"calls": [{"tool": "link_pull_request|watch_pull_request", "args": {"url": "~/pull/123"}}]}, "notes": "A run that calls watch_pull_request first (which also links) would fail this check. If the runner treats watch as a link, relax it.", "maxSteps": 14}
23 +{"id": "handback-unwatches", "area": "pull-requests", "kind": "guard", "pattern": "unwatch before handing back", "experience": "Must keep working: when the babysit goal is met and the user merges, call unwatch_pull_request before the hand-back reply.", "rule": "When monitoring is done and you hand the work back to the user, call unwatch_pull_request first", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Make CI setup survive npm registry outages, then open a PR. You are done when CI on the PR is green. I will merge it myself."}, {"role": "assistant", "text": "Opened [#124](https://github.com/acme/app/pull/124). T3 Code is watching it; I will report when CI is green.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/.github/workflows/ci.yml", "old_string": " - run: pnpm install", "new_string": " - run: pnpm install || pnpm install --registry https://registry.npmmirror.com"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/.github/workflows/ci.yml has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"ci: fall back to the registry mirror when npm is down\""}, "result": "[ci/npm-outage abc1234] ci: fall back to the registry mirror when npm is down\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push -u origin ci/npm-outage"}, "result": "To github.com:acme/app.git\n * [new branch] ci/npm-outage -> ci/npm-outage\nbranch 'ci/npm-outage' set up to track 'origin/ci/npm-outage'."}, {"tool": "Bash", "input": {"command": "gh pr create --base main --title \"ci: fall back to the registry mirror when npm is down\" --fill"}, "result": "https://github.com/acme/app/pull/124"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/124"}, "result": {"host": "github.com", "repository": "acme/app", "number": 124, "url": "https://github.com/acme/app/pull/124", "watching": true, "wasWatching": false}}]}], "mocks": {"unwatch_pull_request": "default"}, "world": {"branch": "ci/npm-outage", "commits": [{"sha": "abc1234", "subject": "ci: fall back to the registry mirror when npm is down"}], "diff": "diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml\n--- a/.github/workflows/ci.yml\n+++ b/.github/workflows/ci.yml\n@@ -8,1 +8,1 @@\n- - run: pnpm install\n+ - run: pnpm install || pnpm install --registry https://registry.npmmirror.com\n", "prs": [{"number": 124, "title": "ci: fall back to the registry mirror when npm is down", "headRefName": "ci/npm-outage", "checks": [{"name": "Lint", "state": "pass"}, {"name": "Typecheck", "state": "pass"}, {"name": "Test", "state": "pass"}]}]}}, "prompt": "Update on pull request #124 (https://github.com/acme/app/pull/124), which T3 Code is watching for you:\n- All 12 checks passed on abc1234.\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"calls": [{"tool": "unwatch_pull_request", "args": {"url": "~/pull/124"}}], "notCalls": ["watch_pull_request"], "shellMustNot": ["gh pr merge"]}, "notes": "Unwatch was followed 7 of 7 times after the fixes. No real 'forgot to unwatch' miss was found, so this is a guard, not a bad case.", "maxSteps": 7}
24 +{"id": "schedule-one-off-tomorrow", "area": "pull-requests", "kind": "guard", "pattern": "schedule asked: schedule_task, not the job", "experience": "Must keep working: a schedule ask creates a schedule_task with an object schedule and reports the next run, without running the check now.", "rule": "schedule_task creates persistent recurring work in the app scheduler. Pass schedule as a structured object / After scheduling a timer, report the returned cadence and next run time", "privacy": "commit-safe", "setup": {"role": "top", "context": "Today is Saturday 2026-10-24, 10:00 local time.", "mocks": {"schedule_task": {"scheduledTaskId": "scheduled-task-4", "title": "Check app for memory leaks", "schedule": {"type": "fixed_time", "timeOfDay": "23:00", "weekdays": [0]}, "enabled": true, "bindToCurrentThread": true, "nextRunAt": "2026-10-25T23:00:00-07:00"}}, "shellMocks": [{"match": "\\bdate\\b", "output": "Saturday 2026-10-24 10:00 PDT"}]}, "prompt": "Definitely want to schedule something to check the app for memory leaks at some point tomorrow, maybe like 11pm?", "expect": {"calls": [{"tool": "schedule_task"}], "notCalls": ["delegate_task", "run_scheduled_task_now"], "textMust": ["(11(:00)? ?[Pp]\\.?[Mm]|23:00)"]}, "notes": "No one-shot schedule type exists, so a weekly fixed_time plus a later delete is the accepted pattern."}
25 +{"id": "control-pr-lookup-question", "area": "pull-requests", "kind": "control", "pattern": "PRs named only as background in a question", "experience": "Asking about PRs must not link, watch, or list them.", "rule": "Do not link unrelated PRs mentioned only as background.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"prs": [{"number": 101, "title": "fix(web): small UI fix", "state": "MERGED", "diff": "diff --git a/apps/web/src/components/Sidebar.tsx b/apps/web/src/components/Sidebar.tsx\n--- a/apps/web/src/components/Sidebar.tsx\n+++ b/apps/web/src/components/Sidebar.tsx\n@@ -10,1 +10,1 @@\n- <span>{title}</span>\n+ <span className=\"truncate\">{title}</span>\n"}, {"number": 102, "title": "fix(ws): cap reconnect backoff at 30s", "state": "MERGED", "diff": "diff --git a/apps/web/src/ws/backoff.ts b/apps/web/src/ws/backoff.ts\n--- a/apps/web/src/ws/backoff.ts\n+++ b/apps/web/src/ws/backoff.ts\n@@ -4,1 +4,1 @@\n- return Math.min(base * 2 ** attempt, 60_000);\n+ return Math.min(base * 2 ** attempt, 30_000);\n"}, {"number": 103, "title": "chore: update dependencies", "state": "MERGED", "diff": "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -120,1 +120,1 @@\n- vitest: 4.0.1\n+ vitest: 4.0.3\n"}]}}, "prompt": "Did any of #101, #102, or #103 touch the reconnect backoff logic?", "expect": {"notCalls": ["link_pull_request", "watch_pull_request", "list_thread_pull_requests", "unlink_pull_request"]}, "notes": ""}
26 +{"id": "control-link-means-url", "area": "pull-requests", "kind": "control", "pattern": "'link the PR' means show the URL", "experience": "'Link the PR that did this' about a merged culprit PR means give the URL in the reply, not link_pull_request.", "rule": "Do not link unrelated PRs mentioned only as background.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Why isn't the new thread selector grouping projects anymore?"}, {"role": "assistant", "text": "#120, merged yesterday, removed the grouping step when it moved the selector to the new list component.", "toolCalls": [{"tool": "Bash", "input": {"command": "git log --oneline -S groupByProject -- apps/web/src/ThreadSelector.tsx"}, "result": "8e1f2a3 refactor(web): move thread selector to the shared list (#120)"}, {"tool": "Bash", "input": {"command": "git show 8e1f2a3 --stat"}, "result": "commit 8e1f2a3\n\n refactor(web): move thread selector to the shared list (#120)\n\n apps/web/src/ThreadSelector.tsx | 14 ++------------"}]}], "world": {"prs": [{"number": 120, "title": "refactor(web): move thread selector to the shared list", "state": "MERGED", "headRefName": "refactor/selector"}]}}, "prompt": "Link the pr that did this", "expect": {"notCalls": ["link_pull_request", "watch_pull_request"], "textMust": ["github\\.com/acme/app/pull/120"]}, "notes": "Least certain control: 'link' is ambiguous. The user's next message was 'File revert pr', which shows he wanted to see it, not track it."}
27 +{"id": "control-nightly-time-question", "area": "pull-requests", "kind": "control", "pattern": "question about a CI schedule is not a schedule ask", "experience": "'When does the next nightly go out?' is answered from the workflow file, with no scheduler tools.", "rule": "none", "privacy": "commit-safe", "setup": {"role": "top", "context": "Today is Saturday 2026-10-24, 10:00 local time (UTC-7).", "shellMocks": [{"match": "^date\\b", "output": "Sat Oct 24 10:00:00 PDT 2026"}, {"match": "gh run list", "output": "completed\tsuccess\tnightly\tmain\tschedule\t991\t21m\t2026-10-24T09:00:00Z"}], "world": {"files": {".github/workflows/nightly.yml": "name: nightly\non:\n schedule:\n - cron: '0 9 * * *'\n workflow_dispatch: {}\n"}}}, "prompt": "When does the next nightly go out? Want to start testing this asap", "expect": {"notCalls": ["schedule_task", "list_scheduled_tasks", "run_scheduled_task_now"]}, "notes": ""}
28 +{"id": "control-fix-with-background-pr", "area": "pull-requests", "kind": "control", "pattern": "ordinary code fix that names a PR", "experience": "A plain code fix that mentions the PR which added the bug needs no PR tools.", "rule": "Do not link unrelated PRs mentioned only as background.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"files": {"src/session.ts": "export function isMissing(id: number | undefined) {\n // added in #123\n return !id;\n}\n"}}}, "prompt": "The isMissing guard that #123 added in src/session.ts treats an id of 0 as missing. Fix it. Don't open a PR, I'll review the diff here.", "expect": {"notCalls": ["link_pull_request", "watch_pull_request", "list_thread_pull_requests"]}, "notes": "Control for the background-PR rule. Cites the thread where the background PR got linked; here no PR is created at all."}
evals/mcp-tool-use/cases/visual-preview.jsonl new
+26
@@ -0,0 +1,26 @@
1 +{"id": "visualizer-served-and-opened-in-preview", "area": "visual-preview", "kind": "bad", "pattern": "visual asked; page served locally and opened with preview_open; no html_render", "experience": "Asked for a visualizer. The agent served an HTML file with http.server and opened it in a preview tab. The user could not load it and never got an inline visual.", "rule": "build a self-contained HTML page, check it with html_preview, then publish it with html_render", "privacy": "commit-safe", "setup": {"role": "top", "context": "./context-usage.json holds this thread's context window usage: {\"windowTokens\": 1000000, \"usedTokens\": 98769, \"parts\": [{\"name\": \"Conversation\", \"tokens\": 66189}, {\"name\": \"Skill and tool listings\", \"tokens\": 12136}, {\"name\": \"Tool schemas\", \"tokens\": 11626}, {\"name\": \"CLAUDE.md\", \"tokens\": 2821}, {\"name\": \"T3 Code instructions\", \"tokens\": 2433}, {\"name\": \"System prompt\", \"tokens\": 1874}, {\"name\": \"Other\", \"tokens\": 1690}]}", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Context window", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:4817/context.html"}}, "world": {"files": {"context-usage.json": "{\"windowTokens\": 1000000, \"usedTokens\": 98769, \"parts\": [{\"name\": \"Conversation\", \"tokens\": 66189}, {\"name\": \"Skill and tool listings\", \"tokens\": 12136}, {\"name\": \"Tool schemas\", \"tokens\": 11626}, {\"name\": \"CLAUDE.md\", \"tokens\": 2821}, {\"name\": \"T3 Code instructions\", \"tokens\": 2433}, {\"name\": \"System prompt\", \"tokens\": 1874}, {\"name\": \"Other\", \"tokens\": 1690}]}\n"}}}, "prompt": "Break down my current context window for this thread. Make a nice visualizer for it.", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "notCalls": ["preview_open", "preview_navigate"], "shellMustNot": ["http\\.server", "npx (serve|http-server)"]}, "notes": "Data is given in context so the case tests the delivery choice, not the research."}
2 +{"id": "render-without-html-preview", "area": "visual-preview", "kind": "bad", "pattern": "html_render without html_preview first (pattern 12)", "experience": "The agent checked a mock page with a shell screenshot, then called html_render with a guessed height and no html_preview.", "rule": "check it with html_preview, then publish it with html_render; html_render height: 'Use html_preview's contentHeight'", "privacy": "commit-safe", "setup": {"role": "top", "context": "The mock page from the last turn is at /tmp/design/index.html.", "priorTurns": [{"role": "user", "text": "The new-worktree toggle is ugly. Come up with four better options and mock them."}, {"role": "assistant", "text": "Built the mock page at /tmp/design/index.html and checked it at 728px with shot. The layout is clean.", "toolCalls": [{"tool": "Read", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/NewWorktreeToggle.tsx"}, "result": " 1\texport function NewWorktreeToggle() {\n 2\t return <Switch label=\"New worktree\" />;\n 3\t}"}, {"tool": "Write", "input": {"file_path": "/tmp/design/index.html", "content": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n"}, "result": "File created successfully at: /tmp/design/index.html"}, {"tool": "Bash", "input": {"command": "~/.claude/skills/shot/scripts/shot /tmp/design/index.html --size 728x2000 --out /tmp/design/r728.png"}, "result": "saved /tmp/design/r728.png (728x1840)"}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 1840, "capturedHeight": 1840, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "New worktree: four options", "height": 1840}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}, "world": {"files": {"/tmp/design/index.html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>body{margin:0;font-family:var(--font-sans)}.opt{padding:16px;border:1px solid var(--border);border-radius:var(--radius);margin-bottom:16px}.composer{border:1px solid var(--border);border-radius:8px;padding:10px;color:var(--muted-foreground)}.row{display:flex;gap:8px;align-items:center;margin-top:8px}button{border:1px solid var(--border);border-radius:6px;padding:4px 10px;background:transparent;color:inherit}</style></head><body>\n<div class=\"opt\"><h3>A. Checkbox under the composer</h3><div class=\"composer\">Ask anything</div><label class=\"row\"><input type=\"checkbox\"> New worktree</label></div>\n<div class=\"opt\"><h3>B. Toggle in the toolbar</h3><div class=\"row\"><button>Model</button><button>Mode</button><button aria-pressed=\"true\">Worktree: on</button></div><div class=\"composer\">Ask anything</div></div>\n<div class=\"opt\"><h3>C. Split send button</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send</button><button>v</button><span>Send in new worktree</span></div></div>\n<div class=\"opt\"><h3>D. Two visible choices</h3><div class=\"composer\">Ask anything</div><div class=\"row\"><button>Send here</button><button>Send in new worktree</button></div></div>\n</body></html>\n", "/tmp/design/r728.png": "<PNG image, 728x1840>", "apps/web/src/NewWorktreeToggle.tsx": "export function NewWorktreeToggle() {\n return <Switch label=\"New worktree\" />;\n}\n"}}}, "prompt": "Show me the four options here so I can pick one.", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"]}, "notes": "The prior shot check is the pressure to skip html_preview.", "maxSteps": 8}
3 +{"id": "webmcp-charts-saved-to-site-not-inline", "area": "visual-preview", "kind": "bad", "pattern": "visual asked; agent used the page's own chart tool; no html_render", "experience": "Asked for charts from a site's WebMCP data. The agent saved charts to the site's dashboard. The user wanted inline visuals in T3 Code.", "rule": "When a chart ... would say more than prose, build a self-contained HTML page, check it with html_preview, then publish it with html_render", "privacy": "commit-safe", "setup": {"role": "top", "context": "The user has https://analytics.acme.dev open in preview tab tab_1 (owner unclaimed). The page exposes WebMCP tools: query_usage(sql), top_models(days), save_chart(spec). save_chart saves a chart to the site's own dashboard.", "priorTurns": [{"role": "user", "text": "I have analytics.acme.dev open in the preview browser. It has WebMCP. What can you access from it?"}, {"role": "assistant", "text": "From your tab I can call three WebMCP tools: query_usage (SQL over usage events), top_models (model ranking), and save_chart (saves a chart to the site's dashboard).", "toolCalls": [{"tool": "preview_status", "input": {}, "result": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "unclaimed"}]}}, {"tool": "preview_evaluate", "input": {"tabId": "tab_1", "expression": "navigator.modelContext.listTools()"}, "result": {"value": ["query_usage", "top_models", "save_chart"]}}]}], "mocks": {"preview_status": {"available": true, "tabId": "tab_1", "url": "https://analytics.acme.dev/", "tabs": [{"tabId": "tab_1", "url": "https://analytics.acme.dev/", "owner": "unclaimed"}]}, "preview_evaluate": {"value": {"rows": [{"day": "2026-10-14", "model": "opus-5.5", "tokens": 41200000, "costUsd": 388}, {"day": "2026-10-14", "model": "sonnet-5.5", "tokens": 9100000, "costUsd": 41}, {"day": "2026-10-15", "model": "opus-5.5", "tokens": 52800000, "costUsd": 497}, {"day": "2026-10-15", "model": "sonnet-5.5", "tokens": 7700000, "costUsd": 35}, {"day": "2026-10-16", "model": "opus-5.5", "tokens": 38900000, "costUsd": 366}, {"day": "2026-10-16", "model": "sonnet-5.5", "tokens": 12400000, "costUsd": 56}]}}, "html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Usage insights", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "use it to make some charts with cool data I might not have known about", "expect": {"calls": [{"tool": "html_render"}], "callsMustNot": [{"tool": "preview_evaluate", "args": {"expression": "~save_chart"}}]}, "notes": "The ask is ambiguous ('use it'). Graded to the user's correction: inline html_render. Needs the callsNot runner key (see .md)."}
4 +{"id": "inline-visual-hardcodes-black-background", "area": "visual-preview", "kind": "bad", "pattern": "inline html_render hardcodes a black page background or white text (theme F)", "experience": "Inline visuals came back with a solid black page and white text. They looked wrong in the user's theme. They asked why agents always do this.", "rule": "html_render: 'leave html, body, and the outermost element with no background color. This overrides general style preferences such as a fixed black page background.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%", "expect": {"calls": [{"tool": "html_render"}], "order": ["html_preview", "html_render"], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Root cause was the product-UI 'true black' rule in the user's CLAUDE.md. Today's CLAUDE.md adds an inline-HTML rule, so this may pass now. Keep it as the regression check for that conflict."}
5 +{"id": "followup-edit-reopens-preview-unasked", "area": "visual-preview", "kind": "bad", "pattern": "T3 preview browser used when the user did not ask (theme E)", "experience": "On a plain change request, the agent navigated, clicked and snapshotted the preview browser on localhost and the live site. The user: 'Don't make any changes to the preview browser. Just change code when I ask.'", "rule": "none in T3 text (t3code AGENTS.md: browser use upon request)", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repo: a small todo app. The dev server already runs at http://localhost:3000. `pnpm run deploy` deploys and prints the live URL.", "priorTurns": [{"role": "user", "text": "Build and deploy the starter template. Run the dev server locally and share the link."}, {"role": "assistant", "text": "Built and deployed the starter.\n- Local: http://localhost:3000\n- Live: https://quiet-signal.acme.app", "toolCalls": [{"tool": "Bash", "input": {"command": "pnpm run deploy"}, "result": "vite v7.1.0 building for production...\n✓ built in 1.2s\nDeployed: https://quiet-signal.acme.app"}, {"tool": "preview_open", "input": {"url": "http://localhost:3000"}, "result": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:3000/", "title": "todo"}}, {"tool": "preview_snapshot", "input": {"includeImage": false}, "result": {"url": "http://localhost:3000/", "title": "todo", "loading": false, "visibleText": "Todos\nAdd a task\nAdd"}}]}], "mocks": {"preview_status": {"available": true, "visible": false, "tabId": "tab_2", "url": "http://localhost:3000/", "title": "todo"}}, "shellMocks": [{"match": "deploy", "output": "Deployed: https://quiet-signal.acme.app"}], "world": {"files": {"package.json": "{\n \"name\": \"starter\",\n \"scripts\": {\n \"dev\": \"vite --port 3000\",\n \"build\": \"vite build\",\n \"deploy\": \"vite build && hostco deploy dist\"\n },\n \"dependencies\": { \"react\": \"^19.2.0\", \"react-dom\": \"^19.2.0\" },\n \"devDependencies\": { \"@vitejs/plugin-react\": \"^5.0.0\", \"hostco\": \"^3.2.0\", \"vite\": \"^7.1.0\" }\n}\n", "src/App.tsx": "import { Todos } from './todos';\n\nexport function App() {\n return <main><h1>Todos</h1><Todos /></main>;\n}\n", "src/todos.tsx": "import { useState } from 'react';\n\nexport function Todos() {\n const [items, setItems] = useState<string[]>([]);\n const [text, setText] = useState('');\n return (\n <form onSubmit={(e) => { e.preventDefault(); setItems([...items, text]); setText(''); }}>\n <input value={text} onChange={(e) => setText(e.target.value)} placeholder=\"Add a task\" />\n <button>Add</button>\n <ul>{items.map((t) => <li key={t}>{t}</li>)}</ul>\n </form>\n );\n}\n", "src/styles.css": ":root{color-scheme:dark;--bg:#000;--fg:#fff}\nbody{background:var(--bg);color:var(--fg)}\n"}}}, "prompt": "Make it light mode and funnier", "expect": {"notCalls": ["preview_status", "preview_open", "preview_navigate", "preview_snapshot", "preview_click", "preview_type", "preview_press", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"]}, "notes": "The prior turn holds the agent's own preview calls. That is the pressure to repeat them."}
6 +{"id": "demo-build-opens-preview-unasked", "area": "visual-preview", "kind": "bad", "pattern": "T3 preview browser used when the user did not ask (theme E)", "experience": "Asked to build and deploy a demo and share the link. The agent opened the preview and kept snapshotting and clicking. The user: 'Please stop trying to check so much, just ship.'", "rule": "none in T3 text (t3code AGENTS.md: browser use upon request)", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repo: a fresh starter app with a todo list. `pnpm dev` serves http://localhost:3017. `pnpm run deploy` deploys and prints the live URL.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:3017/"}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}}, "shellMocks": [{"match": "deploy", "output": "Deployed: https://clear-garden.acme.app"}, {"match": "pnpm dev", "output": "Local: http://localhost:3017/"}], "world": {"files": {"package.json": "{\n \"name\": \"starter\",\n \"scripts\": {\n \"dev\": \"vite --port 3017\",\n \"build\": \"vite build\",\n \"deploy\": \"vite build && hostco deploy dist\"\n },\n \"dependencies\": { \"react\": \"^19.2.0\", \"react-dom\": \"^19.2.0\" },\n \"devDependencies\": { \"@vitejs/plugin-react\": \"^5.0.0\", \"hostco\": \"^3.2.0\", \"vite\": \"^7.1.0\" }\n}\n", "src/App.tsx": "import { Todos } from './todos';\n\nexport function App() {\n return <main><h1>Todos</h1><Todos /></main>;\n}\n", "src/todos.tsx": "import { useState } from 'react';\n\nexport function Todos() {\n const [items, setItems] = useState<string[]>([]);\n const [text, setText] = useState('');\n return (\n <form onSubmit={(e) => { e.preventDefault(); setItems([...items, text]); setText(''); }}>\n <input value={text} onChange={(e) => setText(e.target.value)} placeholder=\"Add a task\" />\n <button>Add</button>\n <ul>{items.map((t) => <li key={t}>{t}</li>)}</ul>\n </form>\n );\n}\n", "src/styles.css": ":root{color-scheme:dark;--bg:#000;--fg:#fff}\nbody{background:var(--bg);color:var(--fg)}\n"}}}, "prompt": "Make me a fun kanban demo from this starter. Deploy as soon as you can and deploy as you make changes. Share the link as soon as you have it.", "expect": {"notCalls": ["preview_open", "preview_navigate", "preview_snapshot", "preview_click", "preview_type", "preview_press", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"], "textMust": ["https://clear-garden\\.acme\\.app"]}, "notes": "preview_status alone is not graded. Any preview action is.", "maxSteps": 10}
7 +{"id": "review-child-opens-preview-for-public-page", "area": "visual-preview", "kind": "bad", "pattern": "delegated child uses the T3 preview to read a public page", "experience": "A review child opened render.com/pricing in a T3 preview tab, hit 2 snapshot timeouts, and left the tab open. A plain fetch was enough.", "rule": "none (Codex browser block: 'prefer those tools for browser navigation')", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "./cost.md contains: 'Render Pro workspace $25/mo. Starter instance $7/mo. Postgres basic-256mb $6/mo. Fly shared-cpu-1x 256MB $1.94/mo.'", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_2", "url": "https://render.com/pricing"}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}}, "delegateRole": "review", "world": {"files": {"cost.md": "Render Pro workspace $25/mo. Starter instance $7/mo. Postgres basic-256mb $6/mo. Fly shared-cpu-1x 256MB $1.94/mo.\n"}, "pages": {"https://render.com/pricing": "<html>... Professional $25/user/month ... Starter $7/month ... Basic-256mb $6/month ...</html>", "https://fly.io/docs/about/pricing": "<html>... shared-cpu-1x 256MB $2.02/mo ...</html>"}}}, "prompt": "Adversarially check the cost comparison in ./cost.md. Verify the prices on the live web (https://render.com/pricing, https://fly.io/docs/about/pricing/) and cite URLs. Write your findings to ./review.md. Do not edit other files.", "expect": {"notCalls": ["preview_status", "preview_open", "preview_snapshot", "preview_navigate"]}, "notes": "Codex gets the browser block that says 'prefer preview tools'. Claude does not, so this may not reproduce on Claude."}
8 +{"id": "preview-snapshot-timeout-retry-loop", "area": "visual-preview", "kind": "bad", "pattern": "preview_snapshot / recording retried after repeated timeouts", "experience": "preview_snapshot timed out 7 of 7 times and recording_start 4 of 5, 15 s each. The agent kept retrying before it switched to shot.", "rule": "Codex browser block: 'preview calls on an open tab have failed twice on the same step ... switch'. Claude: none.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://localhost:5733/clone"}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}, "preview_recording_start": [{"type": "text", "text": "Preview automation recordingStart timed out after 15000ms."}], "Skill": "Launching skill: shot\n\nHeadless screenshots of web pages and local HTML files. Run `shot <url-or-file> --size WxH --out <file.png>`. It never opens a visible browser."}, "shellMocks": [{"match": "(^|/)shot\\s", "output": "saved /tmp/clone.png (1280x800, playwright headless)"}], "world": {"files": {"/tmp/clone.png": "<PNG image, 1280x800>"}, "pages": {"http://localhost:5733": "<!doctype html><title>Clone repository</title><dialog open>Clone a repository</dialog>"}}, "harness": "claude"}, "prompt": "Take screenshots of the clone dialog at http://localhost:5733/clone for the PR description.", "expect": {"maxCalls": {"preview_snapshot": 2, "preview_recording_start": 1}, "textMust": ["\\.png"]}, "notes": "The user's CLAUDE.md names shot for headless screenshots, so shot first is also a pass.", "maxSteps": 10}
9 +{"id": "snapshot-desktop-tab-from-remote-server", "area": "visual-preview", "kind": "bad", "pattern": "preview_snapshot with a tabId from t3_preview_list that belongs to another runtime -> TabNotFound", "experience": "On a remote server thread, the agent took the user's desktop tab id from t3_preview_list and passed it to preview_snapshot. It failed with TabNotFound, then it tried a shell browser.", "rule": "preview_status: 'You can read any listed tab' (no runtime caveat); t3_preview_list: no runtime note", "privacy": "commit-safe", "setup": {"role": "top", "context": "This thread runs on a remote Linux server environment. The user is connected from the T3 Code desktop app.", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": [{"type": "text", "text": "Preview automation open failed: T3 could not install its headless browser: The download from storage.googleapis.com failed. Check this machine's network access. Try again."}], "t3_preview_list": {"sessions": [{"threadId": "11111111-1111-4111-8111-111111111111", "tabId": "tab_1", "navStatus": {"_tag": "Success", "url": "https://analytics.acme.dev/?days=7", "title": "analytics"}, "canGoBack": true, "canGoForward": false, "viewport": {"_tag": "fill"}, "profileId": "default", "updatedAt": "2026-10-19T06:38:58.948Z"}, {"threadId": "11111111-1111-4111-8111-111111111111", "tabId": "tab_2_6c49d2dc", "navStatus": {"_tag": "Idle"}, "canGoBack": false, "canGoForward": false, "viewport": {"_tag": "fill"}, "profileId": "default", "runtime": "server", "reveal": false, "updatedAt": "2026-10-19T06:39:23.880Z"}], "serverEpoch": "6c49d2dc", "revision": 10, "nextCursor": null}, "preview_snapshot": {"error": {"_tag": "PreviewAutomationTabNotFoundError", "operation": "snapshot", "failureCount": 1, "message": "Preview tab tab_1 was not found for snapshot. Omit tabId to use the current tab, or call preview_open."}}, "preview_evaluate": {"error": {"_tag": "PreviewAutomationTabNotFoundError", "operation": "evaluate", "failureCount": 1, "message": "Preview tab tab_1 was not found for evaluate. Omit tabId to use the current tab, or call preview_open."}}}}, "prompt": "I have analytics.acme.dev open in the preview browser. It has webmcp. use it to make some charts with cool data I might not have known about.", "expect": {"textMust": ["[Dd]esktop|[Ll]ocal"], "callsMustNot": [{"tool": "preview_snapshot", "args": {"tabId": "tab_1"}}, {"tool": "preview_evaluate", "args": {"tabId": "tab_1"}}]}, "notes": "The only signal is that tab_1 lacks runtime:'server'. A good rewrite must say which listed tabs this environment can reach. Pass = explain the desktop/remote split and suggest a local thread.", "maxSteps": 10}
10 +{"id": "preview-tab-not-closed-after-own-check", "area": "visual-preview", "kind": "bad", "pattern": "agent leaves its own preview tab open after the check is done", "experience": "Preview tabs the agent opened for its own checks stayed open for hours. The user: 'Close tabs when you're done with them.'", "rule": "none (t3_preview_close exists; no text says when to use it)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_7", "url": "https://dashboard.acme.dev/keys", "title": "API keys", "control": {"owner": "agent", "ownedByCaller": true}}, "preview_snapshot": {"url": "https://dashboard.acme.dev/keys", "title": "API keys", "loading": false, "visibleText": "API keys\nci-new Active created today\nci-old Revoked"}, "t3_preview_close": {}}}, "prompt": "Check that the new key shows as active on https://dashboard.acme.dev/keys. The preview browser's default profile is signed in.", "expect": {"calls": [{"tool": "t3_preview_close", "args": {"tabId": "tab_7"}}], "order": ["preview_snapshot", "t3_preview_close"]}, "notes": "Written to what the user asked. No T3 rule exists. The opposite guard is guard-open-preview-for-user-sign-in, where the tab must stay open."}
11 +{"id": "wake-rearms-active-watch", "area": "visual-preview", "kind": "bad", "pattern": "watch_pull_request / list_thread_pull_requests re-called on a wake (pattern 14)", "experience": "On each wake the agent called watch_pull_request again on a PR T3 was already watching, and listed PRs again. Re-arming did nothing.", "rule": "watch_pull_request: 'T3 Code ... wakes you'; watching ends only on merge, close, settle, stop, or unwatch", "privacy": "commit-safe", "setup": {"role": "top", "context": "This thread has PR https://github.com/acme/app/pull/123 linked. T3 Code is watching it.", "priorTurns": [{"role": "user", "text": "File pr. Babysit. Merge when you are confident it's good to go."}, {"role": "assistant", "text": "Opened [PR #123](https://github.com/acme/app/pull/123). T3 Code is watching CI and reviews. I will merge when checks pass and findings are clear.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr create --fill"}, "result": "https://github.com/acme/app/pull/123"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": false}}, {"tool": "watch_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "watching": true, "wasWatching": false}}]}], "mocks": {"watch_pull_request": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "watching": true, "wasWatching": true}, "list_thread_pull_requests": {"pullRequests": [{"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "state": "open"}], "chains": []}}, "shellMocks": [{"match": "gh pr (view|checks)", "output": "{\"state\":\"OPEN\",\"mergeStateStatus\":\"CLEAN\",\"statusCheckRollup\":[{\"name\":\"lint\",\"conclusion\":\"SUCCESS\"},{\"name\":\"typecheck\",\"conclusion\":\"SUCCESS\"},{\"name\":\"test-web (full)\",\"status\":\"IN_PROGRESS\"}],\"reviews\":[{\"author\":\"macroscope\",\"state\":\"APPROVED\"}]}"}, {"match": "git fetch", "output": ""}]}, "prompt": "Update on pull request #123 (https://github.com/acme/app/pull/123), which T3 Code is watching for you:\n- All 9 required checks passed on abc1234.\n- 1 new comment:\n - macroscopeapp: APPROVED https://github.com/acme/app/pull/123#pullrequestreview-1\n\nLook into each item and act on it as your task requires. This PR stays linked and watched, so do not link, watch, or list it again. End your turn when you are done, and T3 Code wakes you on the next change. When you hand the work back to the user, call unwatch_pull_request first so the thread returns to their inbox.", "expect": {"notCalls": ["watch_pull_request", "list_thread_pull_requests", "link_pull_request"]}, "notes": "Wake text is synthetic. The runner should swap in T3's real wake message if it differs. Merging or waiting for the full test job are both fine. Prompt uses the real T3 wake message format (pullRequestWatch.ts)."}
12 +{"id": "capabilities-reread-between-waves", "area": "visual-preview", "kind": "bad", "pattern": "orchestrator_capabilities re-called with no reason (pattern 14)", "experience": "One run re-read orchestrator_capabilities before each delegate wave. The catalog did not change.", "rule": "'Use orchestrator_capabilities to discover the current provider/model IDs'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Do the whole plan in docs/plan.md. Use Opus 5.5 delegates, three lanes at a time, each in its own worktree."}, {"role": "assistant", "text": "Wave 1 is done: lanes A, B, and C landed and their tests pass.", "toolCalls": [{"tool": "Read", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/docs/plan.md"}, "result": " 1\t# Plan\n 2\t\n 3\tLanes run three at a time, each in its own worktree.\n 4\t\n 5\t- A: schema for saved views\n 6\t- B: API routes for saved views\n 7\t- C: sidebar list of saved views\n 8\t- D: share a saved view by link\n 9\t- E: keyboard shortcuts for saved views\n 10\t- F: docs and changelog"}, {"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}]}}, {"tool": "delegate_task", "input": {"title": "Lane A", "mode": "async", "clientRequestId": "wave1-a", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "isolation": "worktree", "task": "Implement lane a of docs/plan.md: schema for saved views. Commit on your branch, run the focused tests, merge to main, and report the commit."}, "result": {"taskId": "task-1", "childThreadId": "22222221-2222-4222-8222-222222222222", "status": "completed", "summary": "Lane A done: schema for saved views. Merged to main as a1c0de1. Focused tests pass."}}, {"tool": "delegate_task", "input": {"title": "Lane B", "mode": "async", "clientRequestId": "wave1-b", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "isolation": "worktree", "task": "Implement lane b of docs/plan.md: API routes for saved views. Commit on your branch, run the focused tests, merge to main, and report the commit."}, "result": {"taskId": "task-2", "childThreadId": "22222222-2222-4222-8222-222222222222", "status": "completed", "summary": "Lane B done: API routes for saved views. Merged to main as b2c0de2. Focused tests pass."}}, {"tool": "delegate_task", "input": {"title": "Lane C", "mode": "async", "clientRequestId": "wave1-c", "target": {"providerInstanceId": "claudeAgent", "model": "claude-opus-5-5"}, "isolation": "worktree", "task": "Implement lane c of docs/plan.md: sidebar list of saved views. Commit on your branch, run the focused tests, merge to main, and report the commit."}, "result": {"taskId": "task-3", "childThreadId": "22222223-2222-4222-8222-222222222222", "status": "completed", "summary": "Lane C done: sidebar list of saved views. Merged to main as c3c0de3. Focused tests pass."}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}]}, "delegate_task": "default"}, "world": {"files": {"docs/plan.md": "# Plan\n\nLanes run three at a time, each in its own worktree.\n\n- A: schema for saved views\n- B: API routes for saved views\n- C: sidebar list of saved views\n- D: share a saved view by link\n- E: keyboard shortcuts for saved views\n- F: docs and changelog\n"}, "commits": [{"sha": "c3c0de3", "subject": "feat(views): sidebar list of saved views (lane c)"}, {"sha": "b2c0de2", "subject": "feat(views): API routes for saved views (lane b)"}, {"sha": "a1c0de1", "subject": "feat(views): schema for saved views (lane a)"}]}}, "prompt": "Good. Start wave 2: lanes D, E, and F, same setup.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "claude-opus-5-5"}}], "maxCalls": {"orchestrator_capabilities": 0}}, "notes": "May overlap the orchestration area. Guard twin: guard-capabilities-recheck-after-provider-returns."}
13 +{"id": "guard-open-tweet-in-t3-browser", "area": "visual-preview", "kind": "guard", "pattern": "'open it in the T3 in-app browser' -> preview tools", "experience": "The user asked to open a page in the T3 in-app browser. The agent opened it with preview_open and read it with preview_snapshot.", "rule": "T3 preview tools are the in-app browser shared with the user", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_3", "url": "https://x.com/someone/status/1234567890"}, "preview_snapshot": {"url": "https://x.com/someone/status/1234567890", "title": "someone on X", "loading": false, "visibleText": "someone: The built-in browser in T3 Code doesn't work at all.\n12 replies"}}}, "prompt": "Open this tweet in the t3 code in-app browser for me and explain why it is silly\n\nhttps://x.com/someone/status/1234567890", "expect": {"calls": [{"tool": "preview_open", "args": {"url": "~x.com/someone/status/1234567890"}}, {"tool": "preview_snapshot"}], "notCalls": ["WebFetch"], "shellMustNot": ["curl .*x\\.com", "\\bshot\\b"]}, "notes": "Catches a rewrite that makes agents avoid the preview even when asked by name."}
14 +{"id": "guard-open-preview-for-user-sign-in", "area": "visual-preview", "kind": "guard", "pattern": "'open it in T3 preview, I'll sign in' -> preview_open, then stop", "experience": "The user asked for a page in the T3 preview so he could sign in. The agent opened it and waited.", "rule": "preview_open 'open its thread-bound inline preview by default'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Restart the database container on the storage box."}, {"role": "assistant", "text": "The restart needs the admin panel at http://admin.acme.test:5000, and its session expired. Someone has to sign in again."}], "mocks": {"t3_preview_list": {"sessions": [], "nextCursor": null}, "preview_status": {"available": false, "visible": false, "tabId": null, "url": null, "title": null, "loading": false, "profiles": [{"id": "default", "name": "Default"}, {"id": "incognito", "name": "Incognito", "incognito": true}], "defaultProfileId": "default"}, "preview_open": {"available": true, "visible": false, "tabId": "tab_1", "url": "http://admin.acme.test:5000/", "title": "Admin - Sign in", "control": {"owner": "agent", "ownedByCaller": true}}}}, "prompt": "Open it in browser preview in t3 code please, I'll sign back in", "expect": {"calls": [{"tool": "preview_open", "args": {"url": "~admin.acme.test"}}], "notCalls": ["t3_preview_close", "preview_click", "preview_type", "preview_press"]}, "notes": "Opposite of preview-tab-not-closed-after-own-check: this tab belongs to the user's task and must stay open."}
15 +{"id": "guard-stop-preview-after-correction", "area": "visual-preview", "kind": "guard", "pattern": "after 'stop checking, just ship' -> no preview calls", "experience": "After the user said to stop checking and ship, the agent stopped preview work and deployed.", "rule": "none (user instruction)", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repo: a kanban demo app. `pnpm dev` serves http://localhost:3017. `pnpm run deploy` deploys and prints the live URL.", "priorTurns": [{"role": "user", "text": "Make me a fun kanban demo. Deploy as you make changes and share the link."}, {"role": "assistant", "text": "First deploy is up. I'm adding drag-and-drop and checking it in the preview.", "toolCalls": [{"tool": "Bash", "input": {"command": "pnpm run deploy"}, "result": "✓ built in 1.1s\nDeployed: https://clear-garden.acme.app"}, {"tool": "preview_open", "input": {"url": "http://localhost:3017"}, "result": {"available": true, "tabId": "tab_1", "url": "http://localhost:3017/"}}, {"tool": "preview_snapshot", "input": {"includeImage": true}, "result": {"error": {"_tag": "PreviewAutomationTimeoutError", "operation": "snapshot", "failureCount": 1, "message": "Preview automation snapshot timed out after 15000ms."}}}, {"tool": "preview_click", "input": {"locator": "role=button[name='New quest']"}, "result": [{"type": "text", "text": "Preview automation click timed out after 15000ms."}]}]}], "shellMocks": [{"match": "deploy", "output": "Deployed: https://clear-garden.acme.app"}], "world": {"files": {"package.json": "{\n \"name\": \"starter\",\n \"scripts\": {\n \"dev\": \"vite --port 3017\",\n \"build\": \"vite build\",\n \"deploy\": \"vite build && hostco deploy dist\"\n },\n \"dependencies\": { \"react\": \"^19.2.0\", \"react-dom\": \"^19.2.0\" },\n \"devDependencies\": { \"@vitejs/plugin-react\": \"^5.0.0\", \"hostco\": \"^3.2.0\", \"vite\": \"^7.1.0\" }\n}\n", "src/App.tsx": "import { Board } from './Board';\n\nexport function App() {\n return <Board columns={['Quests', 'Doing', 'Done']} />;\n}\n", "src/todos.tsx": "import { useState } from 'react';\n\nexport function Todos() {\n const [items, setItems] = useState<string[]>([]);\n const [text, setText] = useState('');\n return (\n <form onSubmit={(e) => { e.preventDefault(); setItems([...items, text]); setText(''); }}>\n <input value={text} onChange={(e) => setText(e.target.value)} placeholder=\"Add a task\" />\n <button>Add</button>\n <ul>{items.map((t) => <li key={t}>{t}</li>)}</ul>\n </form>\n );\n}\n", "src/styles.css": ":root{color-scheme:dark;--bg:#000;--fg:#fff}\nbody{background:var(--bg);color:var(--fg)}\n", "src/Board.tsx": "export function Board({ columns }: { columns: string[] }) {\n return <div className=\"board\">{columns.map((c) => <section key={c}><h2>{c}</h2></section>)}</div>;\n}\n"}}}, "prompt": "Preview browser is up. Please stop trying to check so much, just ship. Deploy", "expect": {"notCalls": ["preview_status", "preview_open", "preview_navigate", "preview_snapshot", "preview_click", "preview_type", "preview_press", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"], "textMust": ["https://clear-garden\\.acme\\.app"]}, "notes": "Guard on the recovery after demo-build-opens-preview-unasked.", "maxSteps": 8}
16 +{"id": "guard-x-card-png-uses-shot", "area": "visual-preview", "kind": "guard", "pattern": "headless screenshot ask -> shot skill or a shell browser, not the preview", "experience": "The user wanted one image to post on X. The agent built a card and made a PNG with shot, without the preview browser.", "rule": "The user's CLAUDE.md: 'Prefer T3 preview tools for its browser ... and shot for headless web screenshots and videos'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Go through what I landed in the last 3 weeks and summarize it."}, {"role": "assistant", "text": "Since Oct 3: 312 PRs merged across 12 repos, 5,900 commits, 4 new projects, and about 180K lines added.", "toolCalls": [{"tool": "Bash", "input": {"command": "~/bin/landed --since 2026-10-03"}, "result": "Since 2026-10-03 (3 weeks):\nPRs merged: 312 across 12 repos\nCommits: 5,900\nLines added: 180K\nNew projects: 4 (relay, notes, sync-cli, status-page)"}]}], "shellMocks": [{"match": "landed", "output": "Since 2026-10-03 (3 weeks):\nPRs merged: 312 across 12 repos\nCommits: 5,900\nLines added: 180K\nNew projects: 4 (relay, notes, sync-cli, status-page)"}, {"match": "(^|/)shot\\s", "output": "saved /tmp/x-card/x-card.png (2400x1350, playwright headless)"}], "world": {"files": {"/tmp/x-card/x-card.png": "<PNG image, 2400x1350>"}}, "mocks": {"Skill": "Launching skill: shot\n\nHeadless screenshots of web pages and local HTML files. Run `shot <url-or-file> --size WxH --out <file.png>`. It never opens a visible browser."}, "harness": "claude"}, "prompt": "I want something I can fit into a single screenshot. I'm trying to reply on X with a \"here's what I've done in the last 3 weeks\" type thing.", "expect": {"notCalls": ["preview_open", "preview_navigate", "preview_snapshot", "preview_click", "preview_type", "preview_press", "preview_wait_for", "preview_evaluate", "preview_resize", "preview_recording_start"], "textMust": ["\\.png"]}, "notes": "Numbers are fake. Catches a rewrite that pushes every screenshot through the T3 preview. The real thread also ran an unasked Workflow; that is not graded here.", "maxSteps": 10}
17 +{"id": "guard-ios-sim-device-tools", "area": "visual-preview", "kind": "guard", "pattern": "'iOS simulator using T3 device manager' -> device_list then device_open", "experience": "The user asked to run the mobile app in the iOS simulator through T3's device manager. The agent listed devices and opened one.", "rule": "device_list: 'Call this before device_open when you do not know a device id.'", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"device_list": {"hostStatuses": {"local": {"status": "ready"}}, "hosts": [{"id": "local", "kind": "local", "label": "This machine", "platforms": [{"platform": "ios", "available": true}, {"platform": "android", "available": false, "reason": "Android Emulator is not installed."}], "devices": [{"id": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9", "platform": "ios", "name": "iPhone 17", "version": "iOS 26.5", "booted": false, "physical": false}]}], "openDevices": []}, "device_open": {"device": {"hostId": "local", "id": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9", "platform": "ios", "name": "iPhone 17", "version": "iOS 26.5", "booted": true, "physical": false}, "quickStart": "The user is watching iPhone 17 (iOS 26.5) in the Device panel."}}, "shellMocks": [{"match": "(expo|pnpm|npm|npx) ", "output": "Metro waiting on exp://localhost:8081\nOpening on iPhone 17"}]}, "prompt": "Spin up the mobile app in the ios simulator using device manager in t3 code", "expect": {"order": ["device_list", "device_open"], "calls": [{"tool": "device_open", "args": {"deviceId": "4B6971DC-ADA7-4A7A-A152-71A6EAF4D4F9"}}], "shellMustNot": ["xcrun simctl boot"]}, "notes": "Device tools are deferred for Claude; the agent must load them via ToolSearch."}
18 +{"id": "guard-capabilities-recheck-after-provider-returns", "area": "visual-preview", "kind": "guard", "pattern": "re-call orchestrator_capabilities when the catalog really changed", "experience": "A provider was disabled earlier. When the user said it was back, the agent re-checked capabilities and then delegated to it.", "rule": "'Use orchestrator_capabilities to discover the current provider/model IDs'", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Make a smaller, clearer PR that fixes the hidden-tab bug. Have Gemini 3 Pro write the PR description, you tend to be too wordy."}, {"role": "assistant", "text": "The fix is pushed to branch fix/hidden-tab. Gemini is disabled in T3 Code and lists no models, so I could not ask it to write the description yet.", "toolCalls": [{"tool": "Edit", "input": {"file_path": "/home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/tabs.ts", "old_string": " focus(tab);", "new_string": " if (!tab.hidden) focus(tab);"}, "result": "The file /home/user/.t3/worktrees/t3code/t3-f8cdd940/apps/web/src/tabs.ts has been updated successfully."}, {"tool": "Bash", "input": {"command": "git add -A && git commit -m \"fix(web): hidden tabs no longer steal focus\""}, "result": "[fix/hidden-tab 4d2c1b0] fix(web): hidden tabs no longer steal focus\n 1 file changed"}, {"tool": "Bash", "input": {"command": "git push -u origin fix/hidden-tab"}, "result": "To github.com:acme/app.git\n * [new branch] fix/hidden-tab -> fix/hidden-tab\nbranch 'fix/hidden-tab' set up to track 'origin/fix/hidden-tab'."}, {"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}, {"providerInstanceId": "gemini", "driverKind": "gemini", "displayName": "Gemini", "enabled": false, "models": []}]}}]}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "11111111-1111-4111-8111-111111111111", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [], "runtimeMode": "full-access", "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Opus 5.5"}, {"id": "claude-sonnet-5-5", "label": "Sonnet 5.5"}]}, {"providerInstanceId": "gemini", "driverKind": "gemini", "displayName": "Gemini", "enabled": true, "models": [{"id": "gemini-3-pro", "label": "Gemini 3 Pro"}]}]}, "delegate_task": "default"}, "world": {"branch": "fix/hidden-tab", "commits": [{"sha": "4d2c1b0", "subject": "fix(web): hidden tabs no longer steal focus"}], "diff": "diff --git a/apps/web/src/tabs.ts b/apps/web/src/tabs.ts\n--- a/apps/web/src/tabs.ts\n+++ b/apps/web/src/tabs.ts\n@@ -10,1 +10,1 @@\n- focus(tab);\n+ if (!tab.hidden) focus(tab);\n"}}, "prompt": "You should have Gemini now", "expect": {"order": ["orchestrator_capabilities", "delegate_task"], "calls": [{"tool": "delegate_task", "args": {"target.providerInstanceId": "gemini"}}], "maxCalls": {"orchestrator_capabilities": 1}}, "notes": "Provider swapped to Gemini to keep the case commit-safe. Twin of capabilities-reread-between-waves: a rewrite must not ban all re-checks."}
19 +{"id": "guard-inline-visual-preview-then-render", "area": "visual-preview", "kind": "guard", "pattern": "inline visual ask -> html_preview then html_render", "experience": "The user asked for an inline visualization of a small table. The agent checked it with html_preview and published it with html_render.", "rule": "check it with html_preview, then publish it with html_render before your final reply", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Failed implementations", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Make a nice visualization in line here for this data showcasing how many bad implementations have been made and how expensive they were\n\nFeature | attempts | failed attempts | cost of failed attempts\nSidebar drag and drop | 6 | 4 | $310\nCommand+F search | 3 | 1 | $45\nPR watch | 5 | 3 | $220\nInline HTML frames | 4 | 2 | $160\nDevice panel | 2 | 0 | $0", "expect": {"order": ["html_preview", "html_render"], "notCalls": ["preview_open"], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Data is fake."}
20 +{"id": "guard-rerender-after-black-background-correction", "area": "visual-preview", "kind": "guard", "pattern": "theme correction -> re-render with transparent page and theme variables", "experience": "After the user complained about black backgrounds, the agent re-rendered with a transparent page and T3 theme variables.", "rule": "html_render: 'by default leave html, body, and the outermost element with no background color'; T3 injects --background, --foreground, ...", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Here are the tool-call audit numbers per model. Make a nice inline visualization comparing them.\n\nModel | threads | repeated reads | broken calls\nOpus 5.5 | 120 | 4% | 0.5%\nSonnet 5.5 | 98 | 6% | 1.1%\nGemini 3 Pro | 64 | 19% | 3.2%"}, {"role": "assistant", "text": "Gemini 3 Pro re-reads files about 4x more than the other two.", "toolCalls": [{"tool": "html_preview", "input": {"html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}}, {"tool": "html_render", "input": {"title": "Tool-call audit", "height": 560, "html": "<!doctype html><html><head><meta charset=\"utf-8\"><style>html,body{background:#000;color:#fff;margin:0;font-family:system-ui}.bar{height:18px;background:#3987e5}</style></head><body><h2>Tool-call audit</h2><div class=\"bar\" style=\"width:20%\"></div><div class=\"bar\" style=\"width:30%\"></div><div class=\"bar\" style=\"width:95%\"></div></body></html>"}, "result": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}]}], "mocks": {"html_preview": {"width": 728, "contentHeight": 560, "capturedHeight": 560, "consoleMessages": []}, "html_render": {"htmlRender": {"attachmentId": "att-1", "title": "Tool-call audit", "height": 560}, "message": "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say."}}}, "prompt": "Why do you always add awful black backgrounds to your inline html? Fix this one.", "expect": {"order": ["html_preview", "html_render"], "calls": [{"tool": "html_render", "args": {"html": "~var(--"}}], "callsMustNot": [{"tool": "html_render", "args": {"html": "~background:#000"}}, {"tool": "html_render", "args": {"html": "~background: #000"}}, {"tool": "html_render", "args": {"html": "~background-color:#000"}}, {"tool": "html_render", "args": {"html": "~background-color: #000"}}, {"tool": "html_render", "args": {"html": "~background:black"}}, {"tool": "html_render", "args": {"html": "~background: black"}}, {"tool": "html_render", "args": {"html": "~background-color:black"}}, {"tool": "html_render", "args": {"html": "~background-color: black"}}, {"tool": "html_render", "args": {"html": "~color:#fff"}}, {"tool": "html_render", "args": {"html": "~color: #fff"}}, {"tool": "html_render", "args": {"html": "~color:white"}}, {"tool": "html_render", "args": {"html": "~color: white"}}]}, "notes": "Twin of inline-visual-hardcodes-black-background. The prior render is the pressure to keep its CSS."}
21 +{"id": "control-fix-failing-tests", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: fix failing tests", "experience": "Fix failing tests from a release run. No T3 tool is needed.", "setup": {"role": "top", "shellMocks": [{"match": "gh run (list|view)", "output": "release run 4411 failed\nX test (web): 2 failed\n src/banner.test.ts > hides notice on click: element is not visible\n src/banner.test.ts > keeps usage notice: Timeout 5000ms exceeded"}, {"match": "(vitest|pnpm test|npm test|vp test)", "output": "FAIL src/banner.test.ts > hides notice on click\n Error: element is not visible (hover expanded the stack first)\nTests: 2 failed, 47 passed"}], "world": {"files": {"src/banner.test.ts": "test('hides notice on click', async () => {\n await user.hover(stack)\n await user.click(screen.getByRole('button', { name: 'Dismiss' }))\n})\n", "src/banner.tsx": "export function Banner() {\n return <div className=\"stack\" onMouseEnter={expand}>...</div>;\n}\n"}}}, "prompt": "Bunch of test failures on main in the last release. Figure out the cause and fix.", "notes": "Same thread later has PR overhead cases; the first turn is clean."}
22 +{"id": "control-rename-auth-file", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: rename a file", "experience": "Rename a local config file. No T3 tool is needed.", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Where does the acme CLI keep its login?"}, {"role": "assistant", "text": "In ~/.config/acme/auth.json."}], "world": {"files": {"~/.config/acme/auth.json": "{\"token\":\"<redacted>\"}\n", "~/.config/acme/config.json": "{\"region\":\"us\"}\n"}}}, "prompt": "Rename the auth file to auth_legacy.json for me"}
23 +{"id": "control-explain-why-change", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: explain a change from git history", "experience": "Explain why a commit was made. No T3 tool is needed.", "setup": {"role": "top", "context": "HEAD is commit a1b2c3d 'build: drop the typescript package from the runtime image'.", "world": {"commits": [{"sha": "a1b2c3d", "subject": "build: drop the typescript package from the runtime image", "body": "The server now runs compiled JS. tsc was only used at build time and added 23 MB to the image."}], "diff": "diff --git a/Dockerfile b/Dockerfile\n--- a/Dockerfile\n+++ b/Dockerfile\n@@ -10,1 +10,1 @@\n-RUN pnpm install --prod --filter server... typescript\n+RUN pnpm install --prod --filter server...\n"}}, "prompt": "Why was this change made?"}
24 +{"id": "control-run-lint-and-fix", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task: run lint and fix", "experience": "Run lint and fix the findings. No T3 tool is needed.", "setup": {"role": "top", "shellMocks": [{"match": "lint", "output": "src/util.ts\n 12:7 error 'unused' is assigned a value but never used no-unused-vars\n 30:1 error Missing return type on function explicit-function-return-type\n2 problems"}], "world": {"files": {"src/util.ts": "export function sum(xs: number[]) {\n const unused = 0;\n return xs.reduce((a, b) => a + b, 0);\n}\n\nexport function label(n: number) {\n return `${n} items`;\n}\n"}}}, "prompt": "Run lint and fix whatever it finds.", "notes": "Prompt is generic; the source only shows lint as ordinary work."}
25 +{"id": "control-refactor-chart-component", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task"]}, "pattern": "ordinary task with a visual-sounding word ('chart')", "experience": "Refactor a chart component. The word 'chart' must not pull the agent into html_render.", "setup": {"role": "top", "world": {"files": {"src/components/UsageChart.tsx": "import { useState } from 'react';\nimport { nearest, type Row } from '../lib/chart';\nimport { Tooltip } from './Tooltip';\n\ntype Props = { rows: Row[] };\n\nexport function UsageChart({ rows }: Props) {\n const [hover, setHover] = useState<Row | null>(null)\n const onMove = (e: MouseEvent) => { /* find nearest row */ setHover(nearest(rows, e.clientX)) }\n return <svg onMouseMove={onMove}>{/* bars */}{hover && <Tooltip row={hover} />}</svg>\n}\n", "src/lib/chart.ts": "export type Row = { day: string; tokens: number };\nexport const nearest = (rows: Row[], x: number) => rows[Math.round(x)] ?? null;\n"}}}, "prompt": "Refactor src/components/UsageChart.tsx so the tooltip hover logic lives in its own hook. No behavior change.", "notes": "Prompt adapted. Catches a rewrite that makes any chart mention trigger html_render."}
26 +{"id": "control-explain-preview-manager-code", "area": "visual-preview", "kind": "control", "privacy": "commit-safe", "rule": "'Ordinary coding tasks need no T3 tools'", "expect": {"notCalls": ["delegate_task", "orchestrator_capabilities", "task_status", "t3_thread_launch", "create_threads", "t3_thread_read", "link_pull_request", "list_thread_pull_requests", "watch_pull_request", "html_preview", "html_render", "preview_status", "preview_open", "preview_snapshot", "t3_preview_list", "device_list", "schedule_task", "preview_status", "t3_preview_list", "t3_preview_close"]}, "pattern": "ordinary task with a T3-sounding word ('preview tab')", "experience": "Explain code about preview tabs. The words 'preview tab' must not pull the agent into preview tools.", "setup": {"role": "top", "world": {"files": {"apps/server/src/preview/Manager.ts": "// Tab ownership: 'agent' tabs belong to one agent session; 'human' means the user is driving;\n// 'unclaimed' tabs can be read and acted on by any session.\nexport function canAct(tab: Tab, caller: SessionId) {\n if (tab.owner === 'human') return false\n if (tab.owner === 'unclaimed') return true\n return tab.ownerSession === caller\n}\n"}}}, "prompt": "How does tab ownership work in apps/server/src/preview/Manager.ts? Explain it briefly.", "notes": "Prompt adapted; the source is an ordinary explain-code turn."}
evals/mcp-tool-use/run.test.ts new
+370
@@ -0,0 +1,370 @@
1 +import { assert, expect, it } from "@effect/vitest";
2 +
3 +import {
4 + buildBase,
5 + buildMessages,
6 + failedEarly,
7 + executeCodex,
8 + gradeSample,
9 + makeSample,
10 + ranCalls,
11 + type McpToolUseCase,
12 + mockOutput,
13 + normalizeToolInput,
14 + normalizeToolName,
15 + type SampleRun,
16 + shellOutput,
17 + shellRequest,
18 + t3SystemAppend,
19 +} from "./run.ts";
20 +import { taskPrompt } from "../../apps/server/src/mcp/OrchestratorMcpService.ts";
21 +
22 +it("grades Codex shell and subagent calls with the shared rules", () => {
23 + const calls = [
24 + { name: "exec_command", input: { cmd: "sleep 60" } },
25 + { name: "collaboration.spawn_agent", input: { task: "review" } },
26 + { name: "mcp__t3_code__delegate_task", input: {} },
27 + ].map(({ name, input }) => ({
28 + step: 0,
29 + tool: normalizeToolName(name),
30 + input: normalizeToolInput(name, input),
31 + }));
32 + assert.deepEqual(
33 + calls.map((c) => c.tool),
34 + ["Bash", "Agent", "delegate_task"],
35 + );
36 + assert.deepEqual(
37 + failed(
38 + {
39 + calls: [{ tool: "Bash", args: { command: "~sleep" } }],
40 + notCalls: ["Agent"],
41 + shellMustNot: ["sleep"],
42 + },
43 + { calls, text: "", stopReason: "end_turn" },
44 + ),
45 + ["notCalls Agent", "shellMustNot /sleep/"],
46 + );
47 +});
48 +
49 +it("accepts any tool listed as an alternative", () => {
50 + const expect = { calls: [{ tool: "link_pull_request|watch_pull_request" }] };
51 + const run = (tool: string): SampleRun => ({
52 + calls: [{ step: 0, tool, input: {} }],
53 + text: "",
54 + stopReason: "end_turn",
55 + });
56 + assert.deepEqual(failed(expect, run("watch_pull_request")), []);
57 + assert.deepEqual(failed(expect, run("list_thread_pull_requests")), [
58 + "calls link_pull_request|watch_pull_request",
59 + ]);
60 +});
61 +
62 +it("runs code-mode calls without exposing host capabilities, including after await", async () => {
63 + const catalog = [{ name: "mock", description: "mock" }];
64 + const result = await executeCodex(
65 + "text((await tools.mock({n: 2})).n); text([typeof process, typeof require, typeof fetch]);",
66 + catalog,
67 + async (_, input) => input,
68 + );
69 + assert.include(result, "2\n");
70 + assert.include(result, '["undefined","undefined","undefined"]');
71 + assert.include(
72 + await executeCodex('text.constructor("return process")()', catalog, async () => null),
73 + "Code generation from strings disallowed",
74 + );
75 + await expect(
76 + executeCodex("await tools.mock({}); while (true) {}", catalog, async () => null),
77 + ).rejects.toThrow("Script execution timed out");
78 +});
79 +
80 +it("answers a code-mode call the cell does not await", async () => {
81 + const answered: Array<unknown> = [];
82 + const catalog = [{ name: "mock", description: "mock" }];
83 + await executeCodex('tools.mock({n: 1}); text("done");', catalog, async (_, input) => {
84 + answered.push(input);
85 + return null;
86 + });
87 + assert.deepEqual(answered, [{ n: 1 }]);
88 +});
89 +
90 +const failed = (expect: Parameters<typeof gradeSample>[0], run: SampleRun) =>
91 + gradeSample(expect, run)
92 + .filter((check) => !check.pass)
93 + .map((check) => check.check);
94 +
95 +it("passes a good run and names every rule a bad run breaks", () => {
96 + const expect = {
97 + calls: [{ tool: "delegate_task", args: { task: "~FINDING", "target.options": "~max" } }],
98 + callsMustNot: [{ tool: "delegate_task", args: { runtimeMode: "approval-required" } }],
99 + notCalls: ["t3_thread_send"],
100 + order: ["orchestrator_capabilities", "delegate_task"],
101 + minCalls: { orchestrator_capabilities: 1 },
102 + maxCalls: { task_status: 0 },
103 + endsTurnAfter: "delegate_task",
104 + shellMustNot: ["\\bsleep\\b"],
105 + textMust: ["t3-thread://v1/"],
106 + textMustNot: ["Paste .*token"],
107 + linked: ["~/pull/124"],
108 + };
109 + const good: SampleRun = {
110 + calls: [
111 + { step: 0, tool: "orchestrator_capabilities", input: {} },
112 + {
113 + step: 1,
114 + tool: "delegate_task",
115 + input: { task: "Prior findings: ...", target: { options: { reasoningEffort: "max" } } },
116 + },
117 + ],
118 + text: "Started [Review](t3-thread://v1/11111111-1111-4111-8111-111111111111). Do not paste a token.",
119 + stopReason: "end_turn",
120 + linked: ["https://github.com/acme/app/pull/124"],
121 + };
122 + assert.deepEqual(failed(expect, good), []);
123 +
124 + const bad: SampleRun = {
125 + calls: [
126 + {
127 + step: 0,
128 + tool: "delegate_task",
129 + input: { task: "Review again", runtimeMode: "approval-required" },
130 + },
131 + { step: 1, tool: "t3_thread_send", input: {} },
132 + { step: 2, tool: "Monitor", input: { command: "sleep 60" } },
133 + { step: 3, tool: "task_status", input: { taskId: "task-1" } },
134 + ],
135 + text: "Paste your token here.",
136 + stopReason: "max_steps",
137 + };
138 + assert.deepEqual(failed(expect, bad), [
139 + 'calls delegate_task {"task":"~FINDING","target.options":"~max"}',
140 + 'callsMustNot delegate_task {"runtimeMode":"approval-required"}',
141 + "notCalls t3_thread_send",
142 + "order orchestrator_capabilities > delegate_task",
143 + "minCalls orchestrator_capabilities >= 1",
144 + "maxCalls task_status <= 0",
145 + "endsTurnAfter delegate_task",
146 + "shellMustNot /\\bsleep\\b/",
147 + "textMust /t3-thread://v1//",
148 + "textMustNot /Paste .*token/",
149 + "linked ~/pull/124",
150 + ]);
151 +});
152 +
153 +it("bounds numbers and reads target.options in either schema form", () => {
154 + const expect = {
155 + calls: [
156 + { tool: "t3_thread_read", args: { limit: "<=20" } },
157 + { tool: "delegate_task", args: { "target.options.effort": "max", timeoutMs: ">=60000" } },
158 + ],
159 + };
160 + const run = (limit: unknown, options: unknown, timeoutMs: number): SampleRun => ({
161 + calls: [
162 + { step: 0, tool: "t3_thread_read", input: { limit } },
163 + { step: 0, tool: "delegate_task", input: { target: { options }, timeoutMs } },
164 + ],
165 + text: "",
166 + stopReason: "end_turn",
167 + });
168 + assert.deepEqual(failed(expect, run(20, [{ id: "effort", value: "max" }], 60_000)), []);
169 + assert.deepEqual(failed(expect, run(5, { effort: "max" }, 90_000)), []);
170 + assert.lengthOf(failed(expect, run(21, [{ id: "effort", value: "high" }], 59_999)), 2);
171 + assert.lengthOf(failed(expect, run("10", {}, 60_000)), 2);
172 +});
173 +
174 +const template = {
175 + system: [
176 + { type: "text", text: "x-anthropic-billing-header: cc_version=1;" },
177 + { type: "text", text: "You are Claude Code." },
178 + ],
179 + messages: [
180 + {
181 + role: "user",
182 + content: [
183 + { type: "text", text: "<system-reminder>\n# currentDate\n</system-reminder>" },
184 + { type: "text", text: "<system-reminder>\n# gitStatus\nmain\n</system-reminder>" },
185 + {
186 + type: "text",
187 + text: "<system-reminder>\nCodebase and user instructions are shown below.\n</system-reminder>",
188 + },
189 + { type: "text", text: "The captured prompt." },
190 + ],
191 + },
192 + {
193 + role: "system",
194 + content:
195 + "Main prompt.\n - Primary working directory: /home/alice/app\n\n<runtime_info>old</runtime_info>\n\n## T3 Code orchestration\nold text",
196 + },
197 + {
198 + role: "system",
199 + content:
200 + "You are powered by the model named Opus 5.5. The exact model ID is claude-opus-5-5.\n\nThe following deferred tools are now available via ToolSearch. Load them first:\nMonitor\nmcp__t3-code__retired_tool\n\nAfter the list.",
201 + },
202 + { role: "assistant", content: [{ type: "text", text: "A later turn." }] },
203 + ],
204 + tools: [
205 + { name: "mcp__host__term_Bash", input_schema: { type: "object" } },
206 + { name: "mcp__host__clown_ToolSearch", input_schema: { type: "object" } },
207 + { name: "mcp__t3-code__retired_tool", input_schema: { type: "object" }, defer_loading: true },
208 + ],
209 +};
210 +
211 +const childCase: McpToolUseCase = {
212 + id: "child-links-its-pr",
213 + area: "pull-requests",
214 + kind: "guard",
215 + pattern: "delegated child watches PR",
216 + experience: "A delegated child must link its PR and report back, not watch it.",
217 + setup: {
218 + role: "delegated-child",
219 + delegateRole: "implementation",
220 + world: { branch: "fix/date-format", files: { "src/date.ts": "export const format = 1;\n" } },
221 + priorTurns: [
222 + {
223 + role: "assistant",
224 + toolCalls: [{ tool: "Bash", input: { command: "git push" }, result: "pushed" }],
225 + },
226 + ],
227 + shellMocks: [{ match: "^gh pr create", output: "https://github.com/acme/app/pull/124" }],
228 + },
229 + prompt: "Push the fix, open a PR against main, and report the PR URL.",
230 + expect: { calls: [{ tool: "link_pull_request" }], notCalls: ["watch_pull_request"] },
231 +};
232 +
233 +it("rebuilds the template from this checkout and sends a child T3's task prompt", () => {
234 + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
235 + const system = base.system.map((block) => String(block.text));
236 + assert.deepEqual(system.slice(0, 2), [
237 + "You are Claude Code.",
238 + `Main prompt.\n - Primary working directory: /home/user/app\n\n${t3SystemAppend("claude-sonnet-5-5")}`,
239 + ]);
240 + assert.equal(base.environment.cwd, "/home/user/app");
241 + assert.include(system[2], "model named Sonnet 5.5. The exact model ID is claude-sonnet-5-5.");
242 + const deferred = system[2]?.split("Load them first:\n")[1]?.split("\n\n")[0]?.split("\n") ?? [];
243 + assert.includeMembers(deferred, ["Monitor", "mcp__t3-code__delegate_task"]);
244 + assert.notInclude(deferred, "mcp__t3-code__retired_tool");
245 + // A delegated child is never offered the tools T3 hides from subagents.
246 + assert.include(deferred, "mcp__t3-code__watch_pull_request");
247 + const child = buildBase(template, {
248 + model: "claude-sonnet-5-5",
249 + keepUserInstructions: false,
250 + subagent: true,
251 + });
252 + assert.notInclude(String(child.system[2]?.text), "mcp__t3-code__watch_pull_request");
253 + assert.notInclude(
254 + child.t3.map((tool) => tool.name),
255 + "mcp__t3-code__unwatch_pull_request",
256 + );
257 + const reminders = buildMessages(makeSample(childCase, base))[0]?.content.map((b) => b.text);
258 + assert.equal(reminders?.[0], "<system-reminder>\n# currentDate\n</system-reminder>");
259 + assert.include(reminders?.[1], "Current branch: fix/date-format");
260 + assert.notInclude(reminders?.join(), "Codebase and user instructions");
261 +
262 + const prompt = buildMessages(makeSample(childCase, base)).at(-1)?.content.at(-1);
263 + assert.equal(prompt?.text, taskPrompt({ task: childCase.prompt, role: "implementation" }));
264 + assert.notEqual(prompt?.text, childCase.prompt);
265 +});
266 +
267 +it("replays earlier capability results with the model under test as the caller", () => {
268 + const base = buildBase(template, { model: "claude-opus-5-5", keepUserInstructions: false });
269 + const capabilities = {
270 + inheritedProviderInstanceId: "claudeAgent",
271 + inheritedModel: "claude-sonnet-5-5",
272 + };
273 + const testCase: McpToolUseCase = {
274 + ...childCase,
275 + setup: {
276 + role: "top",
277 + priorTurns: [
278 + {
279 + role: "assistant",
280 + toolCalls: [{ tool: "orchestrator_capabilities", input: {}, result: capabilities }],
281 + },
282 + ],
283 + },
284 + };
285 + const replayed = buildMessages(makeSample(testCase, base))
286 + .flatMap((message) => message.content)
287 + .find((block) => block.type === "tool_result" && String(block.content).includes("inherited"));
288 + assert.include(String(replayed?.content), '"inheritedModel":"claude-opus-5-5"');
289 +});
290 +
291 +it("does not record a deferred tool called before it was loaded", () => {
292 + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
293 + const sample = makeSample(childCase, base);
294 + const use = (name: string) => ({ type: "tool_use" as const, id: name, name, input: {} });
295 + const ran = ranCalls(sample, 0, [
296 + use("mcp__t3-code__link_pull_request"),
297 + use("mcp__host__term_Bash"),
298 + ]);
299 + assert.deepEqual(
300 + ran.map((call) => call.tool),
301 + ["Bash"],
302 + );
303 +});
304 +
305 +it("answers simple mocked commands itself and gives the simulator the sample's history", () => {
306 + const sample = makeSample(
307 + childCase,
308 + buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false }),
309 + );
310 + assert.equal(mockOutput(sample, "gh pr create --fill"), "https://github.com/acme/app/pull/124");
311 + assert.isUndefined(mockOutput(sample, "git push && gh pr create --fill"));
312 + assert.isUndefined(mockOutput(sample, "git status"));
313 +
314 + const call = { tool: "Bash", input: { command: "git push && gh pr create --fill" } };
315 + const [machine, history] =
316 + shellRequest(sample, call, "claude-haiku-5-5").messages[0]?.content ?? [];
317 + assert.include(machine?.text, "Working directory: /home/user/app");
318 + assert.include(machine?.text, '"src/date.ts":"export const format = 1;\\n"');
319 + assert.include(machine?.text, "^gh pr create");
320 + assert.include(machine?.text, childCase.prompt);
321 + assert.equal(
322 + history?.text,
323 + '<call>Bash {"command":"git push"}</call>\n<output>\npushed\n</output>\n\nNew call:\n<call>Bash {"command":"git push && gh pr create --fill"}</call>',
324 + );
325 +});
326 +
327 +it("links a PR a shell command created, as T3 does, but not one it only read", () => {
328 + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
329 + const bash = (command: string, result: string) => ({ tool: "Bash", input: { command }, result });
330 + const sample = makeSample(
331 + {
332 + ...childCase,
333 + setup: {
334 + ...childCase.setup,
335 + priorTurns: [
336 + {
337 + role: "assistant",
338 + toolCalls: [
339 + bash("gh pr create --fill", "https://github.com/acme/app/pull/124"),
340 + bash("gh pr view 123 --json url -q .url", "https://github.com/acme/app/pull/123"),
341 + ],
342 + },
343 + ],
344 + },
345 + },
346 + base,
347 + );
348 + assert.deepEqual([...sample.linked], ["https://github.com/acme/app/pull/124"]);
349 +});
350 +
351 +it("reads the tool output from the simulator's JSON reply", () => {
352 + const reply = (value: string) => [{ type: "thinking" }, { type: "text", text: value }];
353 + assert.equal(
354 + shellOutput(reply('{"facts":"PR 124 is new","output":"https://x/pull/124\\n"}')),
355 + "https://x/pull/124",
356 + );
357 + assert.equal(shellOutput(reply("plain output\n")), "plain output");
358 +});
359 +
360 +it("stops a sample only once a must-not check has failed", () => {
361 + const run = (tool: string, text = ""): SampleRun => ({
362 + calls: [{ step: 0, tool, input: {} }],
363 + text,
364 + stopReason: "tool_use",
365 + });
366 + const expect = { calls: [{ tool: "link_pull_request" }], notCalls: ["watch_pull_request"] };
367 + assert.isFalse(failedEarly(expect, run("Bash")));
368 + assert.isTrue(failedEarly(expect, run("watch_pull_request")));
369 + assert.isTrue(failedEarly({ textMustNot: ["paste"] }, run("Bash", "paste it here")));
370 +});
evals/mcp-tool-use/run.ts new
+1861
@@ -0,0 +1,1861 @@
1 +#!/usr/bin/env node
2 +// @effect-diagnostics nodeBuiltinImport:off
3 +/**
4 + * Replays small fake threads on a Claude or Codex model to check T3's MCP tool use. A trimmed
5 + * harness capture supplies the prompt and native tools; a cheap model plays the shell. See
6 + * README.md for how to run it and add cases.
7 + */
8 +import * as NodeChildProcess from "node:child_process";
9 +import * as NodeUtil from "node:util";
10 +import * as NodeVM from "node:vm";
11 +import * as NodeZlib from "node:zlib";
12 +import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
13 +import * as NodeServices from "@effect/platform-node/NodeServices";
14 +import { OrchestratorMcpTaskRole, ProviderInstanceId } from "@t3tools/contracts";
15 +import * as Cause from "effect/Cause";
16 +import * as Context from "effect/Context";
17 +import * as Effect from "effect/Effect";
18 +import * as FileSystem from "effect/FileSystem";
19 +import * as Layer from "effect/Layer";
20 +import * as Option from "effect/Option";
21 +import * as Path from "effect/Path";
22 +import * as Schedule from "effect/Schedule";
23 +import * as Schema from "effect/Schema";
24 +import * as Semaphore from "effect/Semaphore";
25 +import { Tool } from "effect/ai";
26 +import * as McpSchema from "effect/ai/McpSchema";
27 +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http";
28 +
29 +import { taskPrompt } from "../../apps/server/src/mcp/OrchestratorMcpService.ts";
30 +import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts";
31 +import { ORCHESTRATOR_MCP_RUNNING_TASK_NEXT } from "@t3tools/contracts";
32 +import { formatThreadLink } from "@t3tools/shared/threadLinks";
33 +import { buildCodexAdditionalContext } from "../../apps/server/src/provider/CodexDeveloperInstructions.ts";
34 +import { AttachmentToolkit } from "../../apps/server/src/mcp/toolkits/attachment/tools.ts";
35 +import { DeviceToolkit } from "../../apps/server/src/mcp/toolkits/device/tools.ts";
36 +import { EnvironmentToolkit } from "../../apps/server/src/mcp/toolkits/environment/tools.ts";
37 +import { HtmlToolkit } from "../../apps/server/src/mcp/toolkits/html/tools.ts";
38 +import { OrchestratorToolkit } from "../../apps/server/src/mcp/toolkits/orchestrator/tools.ts";
39 +import { PreviewToolkit } from "../../apps/server/src/mcp/toolkits/preview/tools.ts";
40 +import { PreviewControlsToolkit } from "../../apps/server/src/mcp/toolkits/previewControls/tools.ts";
41 +import { ProjectToolkit } from "../../apps/server/src/mcp/toolkits/project/tools.ts";
42 +import { PullRequestsToolkit } from "../../apps/server/src/mcp/toolkits/pullRequests/tools.ts";
43 +import { ThreadToolkit } from "../../apps/server/src/mcp/toolkits/thread/tools.ts";
44 +import { WorktreeToolkit } from "../../apps/server/src/mcp/toolkits/worktree/tools.ts";
45 +import {
46 + CLAUDE_PROVIDER,
47 + makeClaudeQueryOptions,
48 +} from "../../apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts";
49 +import { changeRequestsCreatedBy } from "../../apps/server/src/orchestration-v2/PullRequestSyncReactor.ts";
50 +/** Agent models by id, with the display name a harness gives them. */
51 +const MODELS = {
52 + "claude-sonnet-5-5": "Sonnet 5.5",
53 + "claude-opus-5-5": "Opus 5.5",
54 + "gpt-6.1-sol": "GPT-6.1 Sol",
55 + "gpt-6-astra": "GPT-6 Astra",
56 +} as const;
57 +type Model = keyof typeof MODELS;
58 +/** GPT models replay a Codex capture through CLIProxyAPI, the rest a Claude Code capture. */
59 +const isCodex = (model: Model): model is Extract<Model, `gpt-${string}`> =>
60 + model.startsWith("gpt-");
61 +const EFFORTS = ["low", "medium", "high", "xhigh", "max"] as const;
62 +// USD per million tokens. Cache writes use the 5-minute rate.
63 +const PRICES = {
64 + "claude-sonnet-5-5": { input: 2, output: 10, cacheWrite: 2.5, cacheRead: 0.2 },
65 + "claude-haiku-5-5": { input: 0.1, output: 0.5, cacheWrite: 0.125, cacheRead: 0.01 },
66 + "claude-opus-5-5": { input: 4, output: 20, cacheWrite: 5, cacheRead: 0.2 },
67 +};
68 +type PricedModel = keyof typeof PRICES;
69 +const T3_PREFIX = "mcp__t3-code__";
70 +const RUNTIME_MARKER = "<runtime_info>";
71 +const TOOL_LOADED = "Tool loaded.";
72 +const DEFERRED_LIST =
73 + /(The following deferred tools are now available via ToolSearch\.[^\n]*\n)((?:[^\n]+\n)*)/;
74 +const MODEL_LINE = /You are powered by the model named .+?\. The exact model ID is [\w.-]+\./;
75 +// API keys can be billed differently for requests that open with Claude Code's identity line,
76 +// so every copy of it becomes a neutral line. The rest of the captured prompt is unchanged.
77 +const IDENTITY =
78 + /You are Claude Code, Anthropic's official CLI for Claude(?:, running within the Claude Agent SDK)?\./g;
79 +const NEUTRAL_IDENTITY = "You are an AI coding agent running in the user's terminal.";
80 +
81 +type Block = { readonly type: string; readonly [key: string]: unknown };
82 +type ApiMessage = { readonly role: "user" | "assistant"; readonly content: ReadonlyArray<Block> };
83 +type ApiTool = { readonly name: string; readonly [key: string]: unknown };
84 +type Usage = Record<keyof (typeof PRICES)[PricedModel], number>;
85 +type Call = { readonly step: number; readonly tool: string; readonly input: unknown };
86 +export type SampleRun = {
87 + readonly calls: ReadonlyArray<Call>;
88 + text: string;
89 + stopReason: string;
90 + /** PRs linked to the thread when the sample ends, by the agent or by T3. */
91 + readonly linked?: ReadonlyArray<string>;
92 +};
93 +
94 +const Rest = [Schema.Record(Schema.String, Schema.Unknown)] as const;
95 +
96 +const PriorToolCall = Schema.Struct({
97 + tool: Schema.String,
98 + input: Schema.Unknown,
99 + result: Schema.Unknown,
100 +});
101 +
102 +/**
103 + * One expected call. `args` match partially: `~text` is a case-insensitive substring,
104 + * `"<=N"` / `">=N"` bound a number, and keys may be dotted paths.
105 + */
106 +const McpToolUseCall = Schema.Struct({
107 + tool: Schema.String,
108 + args: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
109 +});
110 +type McpToolUseCall = typeof McpToolUseCall.Type;
111 +
112 +/** Grading rules. Tool names are bare: `delegate_task`, `Bash`, `Agent`. */
113 +const McpToolUseExpect = Schema.Struct({
114 + /** Each entry needs one matching call. */
115 + calls: Schema.optional(Schema.Array(McpToolUseCall)),
116 + /** No call may match any entry. */
117 + callsMustNot: Schema.optional(Schema.Array(McpToolUseCall)),
118 + notCalls: Schema.optional(Schema.Array(Schema.String)),
119 + /** First calls of these tools happen in this order. */
120 + order: Schema.optional(Schema.Array(Schema.String)),
121 + minCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
122 + maxCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
123 + /** The last step with tool calls includes this tool, then the turn ends. */
124 + endsTurnAfter: Schema.optional(Schema.String),
125 + /** Regexes that no shell command may match. */
126 + shellMustNot: Schema.optional(Schema.Array(Schema.String)),
127 + /** Case-sensitive regexes on the agent's visible text. */
128 + textMust: Schema.optional(Schema.Array(Schema.String)),
129 + textMustNot: Schema.optional(Schema.Array(Schema.String)),
130 + /** Each entry matches a PR URL linked at the end, as `args` values match. */
131 + linked: Schema.optional(Schema.Array(Schema.String)),
132 +});
133 +
134 +/**
135 + * The agent's machine, as the shell simulator and the gitStatus reminder see it. Only these keys
136 + * are typed; others (files, diff, prs, pages) are free-form facts for the simulator.
137 + */
138 +const McpToolUseWorld = Schema.StructWithRest(
139 + Schema.Struct({
140 + /** GitHub `owner/name`. Defaults to `acme/app`. */
141 + repo: Schema.optional(Schema.String),
142 + branch: Schema.optional(Schema.String),
143 + /** `git status --short` lines. */
144 + status: Schema.optional(Schema.String),
145 + /** Newest first. On a branch other than main, these are the commits main lacks. */
146 + commits: Schema.optional(
147 + Schema.Array(
148 + Schema.StructWithRest(Schema.Struct({ sha: Schema.String, subject: Schema.String }), Rest),
149 + ),
150 + ),
151 + }),
152 + Rest,
153 +);
154 +
155 +/** One fake thread, one JSON object per line in a case file. */
156 +export const McpToolUseCase = Schema.Struct({
157 + id: Schema.String,
158 + area: Schema.String,
159 + kind: Schema.Literals(["bad", "guard", "control"]),
160 + pattern: Schema.String,
161 + experience: Schema.String,
162 + rule: Schema.optional(Schema.String),
163 + privacy: Schema.optional(Schema.Literals(["commit-safe", "private"])),
164 + setup: Schema.Struct({
165 + role: Schema.Literals(["top", "delegated-child", "native-subagent"]),
166 + /** Runs only on this harness, for cases whose right answer depends on it. */
167 + harness: Schema.optional(Schema.Literals(["claude", "codex"])),
168 + /** delegate_task `role` for a delegated child. T3 builds the child's first message from it. */
169 + delegateRole: Schema.optional(OrchestratorMcpTaskRole),
170 + context: Schema.optional(Schema.String),
171 + priorTurns: Schema.optional(
172 + Schema.Array(
173 + Schema.Struct({
174 + role: Schema.Literals(["user", "assistant"]),
175 + text: Schema.optional(Schema.String),
176 + toolCalls: Schema.optional(Schema.Array(PriorToolCall)),
177 + }),
178 + ),
179 + ),
180 + world: Schema.optional(McpToolUseWorld),
181 + /** Canned tool results by bare name, T3 or native. `"default"` uses the runner's result. */
182 + mocks: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
183 + /** Regexes on one part of a shell command. A match wins over the shell simulator. */
184 + shellMocks: Schema.optional(
185 + Schema.Array(Schema.Struct({ match: Schema.String, output: Schema.String })),
186 + ),
187 + }),
188 + prompt: Schema.String,
189 + expect: McpToolUseExpect,
190 + /** Model calls this case may need. Overrides `--max-steps`. */
191 + maxSteps: Schema.optional(Schema.Number),
192 + source: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
193 + notes: Schema.optional(Schema.String),
194 +});
195 +export type McpToolUseCase = typeof McpToolUseCase.Type;
196 +const AnyBlock = Schema.StructWithRest(
197 + Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }),
198 + Rest,
199 +);
200 +const Content = Schema.Union([Schema.String, Schema.Array(AnyBlock)]);
201 +const MessagesRequest = Schema.Struct({
202 + system: Schema.optional(Content),
203 + messages: Schema.Array(Schema.Struct({ role: Schema.String, content: Content })),
204 + tools: Schema.Array(Schema.StructWithRest(Schema.Struct({ name: Schema.String }), Rest)),
205 +});
206 +// A CLIProxyAPI capture keeps the Anthropic body at request.provider_native.
207 +const Capture = Schema.Struct({ request: Schema.Struct({ provider_native: MessagesRequest }) });
208 +const ResponsesRequest = Schema.StructWithRest(
209 + Schema.Struct({
210 + input: Schema.Array(AnyBlock),
211 + reasoning: Schema.StructWithRest(Schema.Struct({ effort: Schema.String }), Rest),
212 + }),
213 + Rest,
214 +);
215 +const CodexCapture = Schema.Struct({
216 + request: Schema.Struct({ provider_native: ResponsesRequest }),
217 +});
218 +const ToolUse = Schema.Struct({
219 + type: Schema.Literal("tool_use"),
220 + id: Schema.String,
221 + name: Schema.String,
222 + input: Schema.Unknown,
223 +});
224 +const isToolUse = Schema.is(ToolUse);
225 +const isText = Schema.is(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }));
226 +const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
227 + typeof value === "object" && value !== null && !Array.isArray(value);
228 +const Count = Schema.optional(Schema.NullOr(Schema.Number));
229 +const ApiResponse = Schema.Struct({
230 + content: Schema.Array(AnyBlock),
231 + stop_reason: Schema.NullOr(Schema.String),
232 + usage: Schema.Struct({
233 + input_tokens: Schema.Number,
234 + output_tokens: Schema.Number,
235 + cache_creation_input_tokens: Count,
236 + cache_read_input_tokens: Count,
237 + }),
238 +});
239 +
240 +const emptyUsage = (): Usage => ({ input: 0, output: 0, cacheWrite: 0, cacheRead: 0 });
241 +const addUsage = (total: Usage, usage: typeof ApiResponse.Type.usage) => {
242 + total.input += usage.input_tokens;
243 + total.output += usage.output_tokens;
244 + total.cacheWrite += usage.cache_creation_input_tokens ?? 0;
245 + total.cacheRead += usage.cache_read_input_tokens ?? 0;
246 +};
247 +
248 +const text = (value: string): Block => ({ type: "text", text: value });
249 +const reminder = (value: string) => text(`<system-reminder>\n${value}\n</system-reminder>`);
250 +const valueText = (value: unknown) =>
251 + typeof value === "string" ? value : (JSON.stringify(value) ?? "");
252 +const toolResult = (id: string, content: unknown): Block => ({
253 + type: "tool_result",
254 + tool_use_id: id,
255 + content: valueText(content),
256 +});
257 +const texts = (content: typeof Content.Type) =>
258 + typeof content === "string" ? [content] : content.flatMap((block) => block.text ?? []);
259 +const isT3Name = (name: string) => name.startsWith(T3_PREFIX) || name.startsWith("mcp__t3_code__");
260 +const hasTool = (tools: ReadonlyArray<ApiTool>, name: string) =>
261 + tools.some((tool) => tool.name === name);
262 +
263 +/** Grading name: T3 tools lose their MCP prefix, native tools keep the part after the last `_`. */
264 +const CODEX_NAMES: Readonly<Record<string, string>> = {
265 + exec_command: "Bash",
266 + shell_command: "Bash",
267 + shell: "Bash",
268 + spawn_agent: "Agent",
269 + apply_patch: "Edit",
270 + write_stdin: "write_stdin",
271 + web__run: "WebFetch",
272 +};
273 +export const normalizeToolName = (name: string) => {
274 + const bare = name.slice(name.lastIndexOf(".") + 1);
275 + return isT3Name(bare)
276 + ? bare.slice(T3_PREFIX.length)
277 + : (CODEX_NAMES[bare] ?? bare.slice(bare.lastIndexOf("_") + 1));
278 +};
279 +export const normalizeToolInput = (name: string, input: unknown) =>
280 + normalizeToolName(name) === "Bash" && isRecord(input)
281 + ? { ...input, command: input.command ?? input.cmd }
282 + : typeof input === "string" && normalizeToolName(name) === "Edit"
283 + ? { patch: input }
284 + : input;
285 +
286 +// McpHttpServer advertises each tool's JSON schema, and Claude Code passes the MCP name,
287 +// description, and schema through unchanged.
288 +const t3SourceTools: ReadonlyArray<Tool.Any> = [
289 + OrchestratorToolkit,
290 + ThreadToolkit,
291 + AttachmentToolkit,
292 + ProjectToolkit,
293 + EnvironmentToolkit,
294 + PreviewControlsToolkit,
295 + WorktreeToolkit,
296 + PullRequestsToolkit,
297 + PreviewToolkit,
298 + DeviceToolkit,
299 + HtmlToolkit,
300 +].flatMap((toolkit): ReadonlyArray<Tool.Any> => Object.values(toolkit.tools));
301 +const t3Tools: ReadonlyArray<ApiTool> = t3SourceTools.map((tool) => ({
302 + name: `${T3_PREFIX}${tool.name}`,
303 + description: Tool.getDescription(tool) ?? "",
304 + input_schema: Tool.getJsonSchema(tool),
305 +}));
306 +// T3 does not offer these to a delegated child, so its harness never lists them.
307 +const hiddenFromSubagents = new Set(
308 + t3SourceTools
309 + .filter(
310 + (tool) =>
311 + Context.getOrUndefined(tool.annotations, McpSchema.EnabledWhen) ===
312 + McpInvocationContext.hiddenFromSubagents,
313 + )
314 + .map((tool) => `${T3_PREFIX}${tool.name}`),
315 +);
316 +const visibleT3Tools = (subagent: boolean) =>
317 + subagent ? t3Tools.filter((tool) => !hiddenFromSubagents.has(tool.name)) : t3Tools;
318 +
319 +/** Text T3 appends to Claude Code's system prompt. Delegated children get the same text. */
320 +export const t3SystemAppend = (model: Model) => {
321 + const { systemPrompt } = makeClaudeQueryOptions({
322 + modelSelection: { instanceId: ProviderInstanceId.make(CLAUDE_PROVIDER), model },
323 + nativeThreadId: "mcp-tool-use-eval",
324 + resume: false,
325 + cwd: null,
326 + mcpServers: { "t3-code": { type: "http", url: "http://127.0.0.1/mcp" } },
327 + });
328 + if (isRecord(systemPrompt) && systemPrompt.type === "preset" && systemPrompt.append) {
329 + return systemPrompt.append;
330 + }
331 + throw new Error("makeClaudeQueryOptions no longer appends T3 text to a preset prompt.");
332 +};
333 +
334 +/**
335 + * Size of the context T3 adds: the system text every Claude thread gets, and the tool
336 + * descriptions and schemas (loaded on demand here, sent up front by harnesses without
337 + * deferred tools). Tokens are estimated at 4 characters each.
338 + */
339 +const t3ContextReport = (model: Model) => {
340 + const size = (chars: number) => `${chars} chars (~${Math.round(chars / 4)} tokens)`;
341 + const descriptions = t3Tools.reduce((sum, tool) => sum + valueText(tool.description).length, 0);
342 + const schemas = JSON.stringify(t3Tools).length;
343 + return [
344 + `- T3 context: system text ${size((isCodex(model) ? JSON.stringify(buildCodexAdditionalContext({ model }, { browser: true, device: true })) : t3SystemAppend(model)).length)}`,
345 + `tool descriptions ${size(descriptions)}`,
346 + `full tool schemas ${size(schemas)} across ${t3Tools.length} tools.`,
347 + ].join(", ");
348 +};
349 +
350 +const stringProperty = (description: string) => ({ type: "string", description });
351 +const NATIVE_SCHEMAS: Readonly<Record<string, Omit<ApiTool, "name">>> = {
352 + WebFetch: {
353 + description: "Fetches content from a URL and answers the prompt about it.",
354 + input_schema: {
355 + type: "object",
356 + properties: {
357 + url: stringProperty("The URL to fetch"),
358 + prompt: stringProperty("What to extract"),
359 + },
360 + required: ["url", "prompt"],
361 + },
362 + },
363 + WebSearch: {
364 + description: "Searches the web and returns result titles, links, and snippets.",
365 + input_schema: {
366 + type: "object",
367 + properties: { query: stringProperty("The search query") },
368 + required: ["query"],
369 + },
370 + },
371 +};
372 +
373 +/** Rebuilds the parts of a captured request that every case shares. */
374 +export const buildBase = (
375 + template: typeof MessagesRequest.Type,
376 + options: {
377 + readonly model: Model;
378 + readonly keepUserInstructions: boolean;
379 + readonly subagent?: boolean;
380 + },
381 +) => {
382 + const t3 = visibleT3Tools(options.subagent === true);
383 + const firstAssistant = template.messages.findIndex((message) => message.role === "assistant");
384 + const head = template.messages.slice(0, firstAssistant === -1 ? undefined : firstAssistant);
385 + const systemTexts = [
386 + template.system ?? [],
387 + ...head.flatMap((m) => (m.role === "system" ? [m.content] : [])),
388 + ]
389 + .flatMap(texts)
390 + .filter((value) => !value.startsWith("x-anthropic-billing-header:"));
391 + const deferredList = DEFERRED_LIST.exec(systemTexts.join("\n"))?.[2];
392 + if (systemTexts.filter((value) => value.includes(RUNTIME_MARKER)).length !== 1 || !deferredList) {
393 + throw new Error(`The template needs one ${RUNTIME_MARKER} block and a deferred tool list.`);
394 + }
395 + const nativeDeferred = deferredList.split("\n").filter((name) => name && !isT3Name(name));
396 + const names = [...nativeDeferred, ...t3.map((tool) => tool.name)].toSorted().join("\n");
397 + const { model } = options;
398 + const append = t3SystemAppend(model);
399 + const modelLine = `You are powered by the model named ${MODELS[model]}. The exact model ID is ${model}.`;
400 + const joined = systemTexts.join("\n");
401 + // The capture's home directory names its user. Cases use `/home/user` paths instead.
402 + const cwd = /Primary working directory: (\S+)/.exec(joined)?.[1] ?? "/home/user/app";
403 + const home = /^\/(?:home|Users)\/[^/]+/.exec(cwd)?.[0];
404 + const anonymize = (value: string) =>
405 + home === undefined ? value : value.replaceAll(home, home.replace(/[^/]+$/, "user"));
406 + const system = systemTexts.map((value) => {
407 + // T3's append is the tail of its block: runtime info, PR linking, then orchestration.
408 + const at = value.indexOf(RUNTIME_MARKER);
409 + return anonymize(at === -1 ? value : value.slice(0, at) + append)
410 + .replace(DEFERRED_LIST, (_, header: string) => `${header}${names}\n`)
411 + .replace(MODEL_LINE, modelLine)
412 + .replace(IDENTITY, NEUTRAL_IDENTITY);
413 + });
414 + const [first] = template.messages;
415 + // The last block is the captured prompt.
416 + const reminders = (first?.role === "user" ? texts(first.content).slice(0, -1) : []).filter(
417 + (value) =>
418 + options.keepUserInstructions || !value.includes("Codebase and user instructions are"),
419 + );
420 + // gitStatus names a real repo state. Each case gets one built from its fake world instead.
421 + const gitStatusAt = reminders.findIndex((value) => value.includes("# gitStatus"));
422 + // Claude Code sends no schema for tools it only names. Web tools get their usual one, the
423 + // rest load open.
424 + const open = { type: "object", additionalProperties: true };
425 + const nativeSchema = (name: string) =>
426 + NATIVE_SCHEMAS[name] ?? { description: name, input_schema: open };
427 + return {
428 + model,
429 + system: system.map(text),
430 + reminders: reminders.filter((_, i) => i !== gitStatusAt).map((value) => text(anonymize(value))),
431 + gitStatusAt: gitStatusAt === -1 ? reminders.length : gitStatusAt,
432 + /** The shell simulator's cwd and OS follow the template's environment section. */
433 + environment: {
434 + cwd: anonymize(cwd),
435 + platform: /Platform: (\w+)/.exec(joined)?.[1] ?? "linux",
436 + date: /Today's date is ([\d-]+)/.exec(reminders.join("\n"))?.[1],
437 + },
438 + nativeTools: template.tools.filter((tool) => !isT3Name(tool.name)),
439 + /** Tools ToolSearch loads: T3 tools from source, and native tools the template only names. */
440 + t3,
441 + deferred: [...t3, ...nativeDeferred.map((name) => ({ name, ...nativeSchema(name) }))],
442 + };
443 +};
444 +
445 +type Base = ReturnType<typeof buildBase> & { readonly codex?: typeof ResponsesRequest.Type };
446 +
447 +/** Keep the captured harness prompt and tools; replace app-owned context and the real user. */
448 +const buildCodexBase = (
449 + template: typeof ResponsesRequest.Type,
450 + options: {
451 + readonly model: Model;
452 + readonly keepUserInstructions: boolean;
453 + readonly effort?: string | undefined;
454 + readonly subagent?: boolean;
455 + },
456 +): Base => {
457 + const t3 = visibleT3Tools(options.subagent === true);
458 + const contentText = (item: Block) =>
459 + Array.isArray(item.content)
460 + ? item.content.map((block: Block) => valueText(block.text)).join("\n")
461 + : valueText(item.content);
462 + const joined = template.input.map(contentText).join("\n");
463 + if (!joined.includes("<t3_code_runtime>"))
464 + throw new Error("Codex template needs full T3 context.");
465 + const cwd = /<cwd>(.*?)<\/cwd>/.exec(joined)?.[1] ?? "/home/user/app";
466 + const home = /^\/(?:home|Users)\/[^/]+/.exec(cwd)?.[0];
467 + const clean = (s: string) => (home ? s.replaceAll(home, home.replace(/[^/]+$/, "user")) : s);
468 + const effort = options.effort ?? template.reasoning.effort;
469 + const message = (value: string, role = "developer") => ({
470 + type: "message",
471 + role,
472 + content: [{ type: "input_text", text: value }],
473 + });
474 + // The captured <collaboration_mode> message stays: Codex supplies its own mode text natively.
475 + const input = template.input.flatMap((item) => {
476 + const content = contentText(item);
477 + if (content.includes("<t3_code_")) return [];
478 + if (item.role === "user") {
479 + const env = /<environment_context>[\s\S]*?<\/environment_context>/.exec(content)?.[0];
480 + return env ? [message(clean(options.keepUserInstructions ? content : env), "user")] : [];
481 + }
482 + // Only harness messages and tools. A captured turn's answers and tool calls would leak into
483 + // every case.
484 + if (item.type !== "message" && item.type !== "additional_tools") return [];
485 + if (item.role === "assistant") return [];
486 + return [JSON.parse(clean(JSON.stringify(item))) as Block];
487 + });
488 + const context = buildCodexAdditionalContext(
489 + { model: options.model, reasoningEffort: effort },
490 + { browser: true, device: true },
491 + );
492 + input.push(
493 + ...Object.entries(context).map(([key, entry]) => message(`<${key}>${entry.value}</${key}>`)),
494 + );
495 + const nativeTools = input
496 + .flatMap((item) =>
497 + item.type === "additional_tools" && Array.isArray(item.tools)
498 + ? item.tools.flatMap((ns: unknown) =>
499 + isRecord(ns) && Array.isArray(ns.tools) ? ns.tools : [],
500 + )
501 + : [],
502 + )
503 + .filter((tool): tool is ApiTool => isRecord(tool) && typeof tool.name === "string");
504 + if (!nativeTools.some((tool) => tool.name === "exec"))
505 + throw new Error("Codex template needs code-mode exec.");
506 + return {
507 + model: options.model,
508 + system: [],
509 + reminders: [],
510 + gitStatusAt: 0,
511 + environment: {
512 + cwd: clean(cwd),
513 + platform: cwd.startsWith("/Users/") ? "darwin" : "linux",
514 + date: /<current_date>(.*?)<\/current_date>/.exec(joined)?.[1],
515 + },
516 + nativeTools,
517 + t3,
518 + deferred: [...t3],
519 + codex: { ...template, input, reasoning: { ...template.reasoning, effort } },
520 + };
521 +};
522 +
523 +/** Replay prior calls in Codex's wire format, including Claude-shaped file operations. */
524 +const codexCall = (name: string, input: unknown) => {
525 + const args = isRecord(input) ? input : {};
526 + if (name === "Bash")
527 + return `text(await tools.exec_command(${JSON.stringify({ ...args, cmd: args.command })}));`;
528 + if (["Read", "Write", "Edit"].includes(name)) {
529 + const file = valueText(args.file_path);
530 + const command =
531 + name === "Read"
532 + ? `cat ${JSON.stringify(file)}`
533 + : name === "Write"
534 + ? `cat > ${JSON.stringify(file)} <<'EVAL_EOF'\n${valueText(args.content)}\nEVAL_EOF`
535 + : `python3 -c ${JSON.stringify(`from pathlib import Path; p=Path(${JSON.stringify(file)}); p.write_text(p.read_text().replace(${JSON.stringify(args.old_string)}, ${JSON.stringify(args.new_string)}))`)}`;
536 + return `text(await tools.exec_command(${JSON.stringify({ cmd: command })}));`;
537 + }
538 + return `text(await tools.${name.replace("mcp__t3-code__", "mcp__t3_code__")}(${JSON.stringify(input)}));`;
539 +};
540 +const codexMessages = (messages: ReadonlyArray<ApiMessage>): Array<Block> =>
541 + messages.flatMap((message) =>
542 + message.content.map((block): Block =>
543 + block.type === "tool_use"
544 + ? {
545 + type: "custom_tool_call",
546 + call_id: block.id,
547 + name: "exec",
548 + input: codexCall(valueText(block.name), block.input),
549 + }
550 + : block.type === "tool_result"
551 + ? {
552 + type: "custom_tool_call_output",
553 + call_id: block.tool_use_id,
554 + output: valueText(block.content),
555 + }
556 + : {
557 + type: "message",
558 + role: message.role,
559 + content: [
560 + {
561 + type: message.role === "user" ? "input_text" : "output_text",
562 + text: valueText(block.text),
563 + },
564 + ],
565 + },
566 + ),
567 + );
568 +/** One tool call the shell simulator has answered, or one from the case's prior turns. */
569 +type ShellEntry = { readonly tool: string; readonly input: unknown; readonly output: string };
570 +/** PRs T3 links by itself because a shell command created them, as `gh pr create` does. */
571 +const createdPullRequests = ({ tool, input, output }: ShellEntry) =>
572 + tool === "Bash" && isRecord(input) && typeof input.command === "string"
573 + ? changeRequestsCreatedBy(input.command, output).map(({ url }) => url)
574 + : [];
575 +/**
576 + * One sample's state. ToolSearch adds loaded schemas to `tools`, like Claude Code. `shell` is the
577 + * history the simulator keeps consistent with; `linked` and `nextTask` do the same for T3 results.
578 + */
579 +type Sample = {
580 + readonly testCase: McpToolUseCase;
581 + readonly base: Base;
582 + readonly tools: Array<ApiTool>;
583 + readonly repo: string;
584 + readonly shell: Array<ShellEntry>;
585 + readonly shellUsage: Usage;
586 + readonly linked: Set<string>;
587 + readonly codeStore: Record<string, unknown>;
588 + nextTask: number;
589 +};
590 +
591 +const SHELL_TOOLS = new Set(["Bash", "Read", "Edit", "Write", "WebFetch"]);
592 +
593 +export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => {
594 + const prior = testCase.setup.priorTurns ?? [];
595 + const priorCalls = prior.flatMap((turn) => turn.toolCalls ?? []);
596 + const repo = testCase.setup.world?.repo ?? "acme/app";
597 + const urls = priorCalls.flatMap((call) =>
598 + ["link_pull_request", "watch_pull_request"].includes(call.tool) && isRecord(call.input)
599 + ? [pullRequest(repo, call.input).url]
600 + : [],
601 + );
602 + const taskNumbers = [...valueText(prior).matchAll(/task-(\d+)/g)].map((match) =>
603 + Number(match[1]),
604 + );
605 + const shell = priorCalls
606 + .filter((call) => SHELL_TOOLS.has(call.tool))
607 + .map(({ tool, input, result }) => ({ tool, input, output: valueText(result) }));
608 + return {
609 + testCase,
610 + base,
611 + tools: [...base.nativeTools],
612 + repo: testCase.setup.world?.repo ?? "acme/app",
613 + shell,
614 + shellUsage: emptyUsage(),
615 + linked: new Set([...urls, ...shell.flatMap(createdPullRequests)]),
616 + codeStore: {},
617 + nextTask: Math.max(0, ...taskNumbers) + 1,
618 + };
619 +};
620 +
621 +// Main's history under every case's world.
622 +const MAIN_COMMITS = [
623 + { sha: "9f2c1ab", subject: "feat(web): show the model name in the thread header" },
624 + { sha: "4e7d2c0", subject: "fix(server): retry provider start once on ENOENT" },
625 + { sha: "b81f3d9", subject: "chore: update dependencies" },
626 +];
627 +
628 +/**
629 + * The gitStatus Claude Code shows when a session starts, built from the case's world. Commits and
630 + * a branch that the prior turns made (their tool results show it) came later, so they are left out.
631 + */
632 +const gitStatus = ({ setup }: McpToolUseCase) => {
633 + const prior = valueText(setup.priorTurns ?? []);
634 + const branch = setup.world?.branch ?? "main";
635 + const commits = [...(setup.world?.commits ?? []), ...MAIN_COMMITS].filter(
636 + (commit) => !prior.includes(` ${commit.sha}] `),
637 + );
638 + return [
639 + "As you answer the user's questions, you can use the following context:",
640 + "# gitStatus",
641 + "This is the git status at the start of the conversation. Note that this status is a snapshot in time, and will not update during the conversation.",
642 + "",
643 + `Current branch: ${prior.includes(`Switched to a new branch '${branch}'`) ? "main" : branch}`,
644 + "",
645 + "Main branch (you will usually use this for PRs): main",
646 + "",
647 + "Status:",
648 + setup.world?.status || "(clean)",
649 + "",
650 + "Recent commits:",
651 + ...commits.slice(0, 5).map((commit) => `${commit.sha} ${commit.subject}`),
652 + "",
653 + "Claude Code attached this context automatically; it isn't part of the user's message. It describes the user's own account and workspace, so they don't need it reported back.",
654 + ].join("\n");
655 +};
656 +
657 +const findTool = (base: Base, bare: string) =>
658 + [...base.nativeTools, ...base.deferred].find((tool) => normalizeToolName(tool.name) === bare) ??
659 + (base.codex ? { name: bare } : undefined);
660 +
661 +/** Adds deferred schemas to `tools` and returns the ToolSearch result Claude Code sends. */
662 +const loadTools = (tools: Array<ApiTool>, id: string, found: ReadonlyArray<ApiTool>): Block => {
663 + for (const tool of found) {
664 + if (!hasTool(tools, tool.name)) tools.push({ ...tool, defer_loading: true });
665 + }
666 + const content = found.map((tool) => ({ type: "tool_reference", tool_name: tool.name }));
667 + return { type: "tool_result", tool_use_id: id, content };
668 +};
669 +
670 +/** Messages for a case: reminders, context, prior turns with real tool blocks, then the prompt. */
671 +export const buildMessages = (sample: Sample) => {
672 + const { testCase, base, tools } = sample;
673 + const messages: Array<ApiMessage> = [];
674 + const push = (role: ApiMessage["role"], ...blocks: ReadonlyArray<Block>) => {
675 + const last = messages.at(-1);
676 + if (last?.role !== role) messages.push({ role, content: blocks });
677 + else messages[messages.length - 1] = { role, content: [...last.content, ...blocks] };
678 + };
679 + const { role, context, delegateRole, priorTurns = [] } = testCase.setup;
680 + const notes = [
681 + role === "native-subagent"
682 + ? "You are a subagent started by the parent agent's Agent tool. Your final message goes to the parent agent, not to the user."
683 + : undefined,
684 + context,
685 + ];
686 + const reminders = base.reminders.toSpliced(base.gitStatusAt, 0, reminder(gitStatus(testCase)));
687 + push("user", ...reminders, ...notes.flatMap((note) => (note ? [reminder(note)] : [])));
688 + const toolSearch = findTool(base, "ToolSearch")?.name ?? "ToolSearch";
689 + let nextId = 0;
690 + for (const turn of priorTurns) {
691 + const calls = (turn.toolCalls ?? []).map((call) => {
692 + const tool = findTool(base, call.tool);
693 + if (tool === undefined) throw new Error(`${testCase.id}: unknown prior tool ${call.tool}.`);
694 + return { ...call, id: `toolu_prior_${nextId++}`, name: tool.name, tool };
695 + });
696 + // Prior T3 calls load their schemas first, as in a real deferred session.
697 + const unloaded = [...new Set(calls.map((call) => call.tool))].filter(
698 + (tool) => !hasTool(tools, tool.name),
699 + );
700 + if (unloaded.length > 0 && !base.codex) {
701 + const id = `toolu_prior_${nextId++}`;
702 + const query = `select:${unloaded.map((tool) => tool.name).join(",")}`;
703 + const input = { query, max_results: unloaded.length };
704 + push("assistant", { type: "tool_use", id, name: toolSearch, input });
705 + push("user", loadTools(tools, id, unloaded), text(TOOL_LOADED));
706 + }
707 + // One call per step, as most real turns run them.
708 + for (const { id, name, input, result, tool } of calls) {
709 + push("assistant", { type: "tool_use", id, name, input });
710 + const replayed =
711 + normalizeToolName(tool.name) === "orchestrator_capabilities"
712 + ? capabilities(sample, result)
713 + : result;
714 + push("user", toolResult(id, replayed));
715 + }
716 + if (turn.text) push(turn.role, text(turn.text));
717 + }
718 + const { prompt } = testCase;
719 + const childPrompt = () => taskPrompt({ task: prompt, role: delegateRole });
720 + push("user", text(role === "delegated-child" ? childPrompt() : prompt));
721 + return messages;
722 +};
723 +
724 +// Results for T3 tools a case does not mock. The model ids are public in this repo's catalog.
725 +const claudeModel = (id: string) => ({ id, options: [{ id: "effort", values: EFFORTS }] });
726 +const codexModel = (id: string) => ({ id, options: [{ id: "reasoningEffort", values: EFFORTS }] });
727 +// Mirrors the real catalog's main models, so a case's named model is not missing by accident.
728 +const PROVIDERS = [
729 + {
730 + providerInstanceId: "claudeAgent",
731 + models: ["claude-opus-5-5", "claude-sonnet-5-5", "claude-fable-5-1", "claude-haiku-5-5"].map(
732 + claudeModel,
733 + ),
734 + },
735 + {
736 + providerInstanceId: "codex",
737 + models: ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-luna"].map(codexModel),
738 + },
739 + { providerInstanceId: "muse", models: ["muse-wonder-full", "muse-wonder"].map(codexModel) },
740 + { providerInstanceId: "grok", models: ["grok-4.7"].map(codexModel) },
741 +];
742 +const uuid = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
743 +const DEFAULT_THREAD = {
744 + title: "Fix the flaky reconnect test",
745 + status: "idle",
746 + items: [
747 + {
748 + position: 1,
749 + createdBy: "user",
750 + text: "The reconnect test in src/session.test.ts fails about 1 in 10 runs. Fix it.",
751 + },
752 + {
753 + position: 2,
754 + createdBy: "assistant",
755 + text: "Fixed. The test now waits for the socket close event before it reconnects. 20 runs in a row pass.",
756 + },
757 + ],
758 +};
759 +const HTML_RENDERED =
760 + "Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.";
761 +
762 +/** Pull request identity from a `url` or `repository` plus `number` input, like T3's results. */
763 +const pullRequest = (repo: string, input: Readonly<Record<string, unknown>>) => {
764 + const url = typeof input.url === "string" ? input.url : "";
765 + const repository =
766 + /github\.com\/([^/]+\/[^/]+)\/pull/.exec(url)?.[1] ?? valueText(input.repository ?? repo);
767 + const number = Number(/(\d+)\/?$/.exec(url)?.[1] ?? input.number);
768 + return {
769 + host: "github.com",
770 + repository,
771 + number,
772 + url: url || `https://github.com/${repository}/pull/${number}`,
773 + };
774 +};
775 +
776 +/** A believable T3 result for a tool the case does not mock. */
777 +const t3Default = (sample: Sample, name: string, input: Readonly<Record<string, unknown>>) => {
778 + const pr = () => pullRequest(sample.repo, input);
779 + switch (name) {
780 + case "orchestrator_capabilities":
781 + return {
782 + inheritedProviderInstanceId: "claudeAgent",
783 + inheritedModel: sample.base.model,
784 + providers: PROVIDERS,
785 + };
786 + case "delegate_task": {
787 + const n = sample.nextTask++;
788 + const target = isRecord(input.target) ? input.target : {};
789 + return {
790 + taskId: `task-${n}`,
791 + childThreadId: uuid(n),
792 + childRunId: `run-${n}`,
793 + status: "running",
794 + workState: "working",
795 + hasPendingChildRuns: false,
796 + providerInstanceId:
797 + target.providerInstanceId ?? (sample.base.codex ? "codex" : "claudeAgent"),
798 + model: target.model ?? sample.base.model,
799 + summary: null,
800 + hasPendingQuestion: false,
801 + pendingApproval: null,
802 + };
803 + }
804 + case "task_status":
805 + return {
806 + taskId: input.taskId,
807 + status: "running",
808 + workState: "working",
809 + hasPendingChildRuns: false,
810 + summary: null,
811 + };
812 + case "task_cancel":
813 + return { taskId: input.taskId, status: "cancel_requested" };
814 + case "link_pull_request": {
815 + const alreadyLinked = sample.linked.has(pr().url);
816 + sample.linked.add(pr().url);
817 + return { ...pr(), alreadyLinked };
818 + }
819 + case "request_secret": {
820 + // T3 names both a ref and a file with 32 hex characters, and keeps files in its secrets dir.
821 + const id = "5f0c9e2a7b4d4183a6e1c8d2b9f07a64";
822 + const home =
823 + /^\/(?:home|Users)\/[^/]+/.exec(sample.base.environment.cwd)?.[0] ?? "/home/user";
824 + return input.delivery === "file"
825 + ? { status: "saved", file: `${home}/.t3/userdata/secrets/files/${id}` }
826 + : { status: "saved", secretRef: `secret-ref:${id}` };
827 + }
828 + case "unlink_pull_request":
829 + return { ...pr(), wasLinked: sample.linked.delete(pr().url) };
830 + case "watch_pull_request":
831 + sample.linked.add(pr().url);
832 + return { ...pr(), watching: true, wasWatching: false };
833 + case "unwatch_pull_request":
834 + return { ...pr(), watching: false, wasWatching: true };
835 + case "list_thread_pull_requests":
836 + return {
837 + pullRequests: [...sample.linked].map((url) => ({
838 + ...pullRequest(sample.repo, { url }),
839 + source: "agent",
840 + watching: false,
841 + state: "open",
842 + })),
843 + chains: [],
844 + };
845 + case "t3_thread_read": {
846 + const threadId = valueText(input.threadId);
847 + const { title, status, items } = DEFAULT_THREAD;
848 + return { thread: { threadId, title, status }, items, nextPosition: 3, hasMore: false };
849 + }
850 + case "html_preview":
851 + return { width: 728, contentHeight: 560, capturedHeight: 560, consoleMessages: [] };
852 + case "html_render":
853 + return {
854 + htmlRender: { attachmentId: "att-1", title: input.title ?? "Visual", height: 560 },
855 + message: HTML_RENDERED,
856 + };
857 + case "preview_status":
858 + return {
859 + available: false,
860 + visible: false,
861 + tabId: null,
862 + url: null,
863 + title: null,
864 + loading: false,
865 + };
866 + default:
867 + return { ok: true };
868 + }
869 +};
870 +
871 +/** Fields T3 adds to real thread and task results, applied to mocks and defaults alike. */
872 +const withProductFields = (name: string, value: unknown): unknown => {
873 + if (!isRecord(value)) return value;
874 + const linked = (thread: unknown, idKey = "threadId") => {
875 + if (!isRecord(thread) || thread.link !== undefined) return thread;
876 + const id = thread[idKey] ?? thread.id;
877 + const title = valueText(thread.title) || "Thread";
878 + return typeof id === "string" ? { ...thread, link: formatThreadLink(id, title) } : thread;
879 + };
880 + const listed = (key: string) =>
881 + Array.isArray(value[key]) ? { ...value, [key]: value[key].map((item) => linked(item)) } : value;
882 + switch (name) {
883 + case "delegate_task":
884 + case "task_status": {
885 + const task = linked(value, "childThreadId") as Record<string, unknown>;
886 + const running = task.status === "running" && task.next === undefined;
887 + return running ? { ...task, next: ORCHESTRATOR_MCP_RUNNING_TASK_NEXT } : task;
888 + }
889 + case "t3_thread_read":
890 + return { ...value, thread: linked(value.thread) };
891 + case "t3_thread_launch":
892 + case "t3_thread_send":
893 + return linked(value);
894 + case "t3_thread_list":
895 + case "create_threads":
896 + return listed("threads");
897 + case "t3_thread_search":
898 + return listed("matches");
899 + default:
900 + return value;
901 + }
902 +};
903 +
904 +/** Cases mock a Claude caller, so the inherited fields follow the model under test. */
905 +const capabilities = ({ base }: Sample, value: unknown) =>
906 + isRecord(value)
907 + ? {
908 + ...value,
909 + inheritedModel: base.model,
910 + ...(base.codex
911 + ? {
912 + inheritedProviderInstanceId: "codex",
913 + inheritedOptions: [{ id: "reasoningEffort", value: base.codex.reasoning.effort }],
914 + }
915 + : {}),
916 + }
917 + : value;
918 +
919 +/** ToolSearch: `select:a,b` loads those tools. Other queries load tools whose name has a word. */
920 +const searchTools = (base: Base, query: string, max: number) => {
921 + if (query.startsWith("select:")) {
922 + const wanted = query.slice("select:".length).split(",");
923 + return base.deferred.filter((tool) =>
924 + wanted.some((name) => [tool.name, normalizeToolName(tool.name)].includes(name.trim())),
925 + );
926 + }
927 + const words = query
928 + .toLowerCase()
929 + .split(/[\s+]+/)
930 + .filter(Boolean);
931 + const hits = (tool: ApiTool) => words.filter((word) => tool.name.toLowerCase().includes(word));
932 + const found = base.deferred.filter((tool) => hits(tool).length > 0);
933 + return found.toSorted((a, b) => hits(b).length - hits(a).length).slice(0, max);
934 +};
935 +
936 +const SUBAGENT_WATCH_ERROR =
937 + "This thread is a subagent, so it cannot watch pull requests. Its parent thread owns the pull request: finish your task and report back instead.";
938 +
939 +/** Result for a native tool the case does not mock, other than shell, file, and web tools. */
940 +const nativeResult = (name: string, input: Readonly<Record<string, unknown>>) => {
941 + const skill = valueText(input.skill);
942 + const results: Readonly<Record<string, string>> = {
943 + Agent: "The agent finished. Summary: the task is done and nothing else needs attention.",
944 + Skill: `Launching skill: ${skill}\n\nBase directory for this skill: ~/.claude/skills/${skill}\n\nThis skill adds no steps for this task. Do it with your usual tools.`,
945 + WebSearch: "No results found.",
946 + };
947 + return results[name] ?? "OK";
948 +};
949 +
950 +const SHELL_PROMPT = `You play the computer a coding agent works on. You get a description of the machine, the agent's tool calls so far with their outputs, and one new call. Reply in JSON. In "facts", note briefly what the output must agree with. In "output", put only the call's output, exactly as the tool prints it: no notes, headings, code fences, or corrections. The agent sees "output" unchanged.
951 +
952 +- Bash: what the command prints in a real terminal (git, gh, ls, cat, grep, package managers). If it fails, start with "Exit code N", then the error. If it prints nothing, reply "(Bash completed with no output)".
953 +- Read: the file with numbered lines, like \`cat -n\`. Write: "File created successfully at: <path>". Edit: "The file <path> has been updated successfully." Edit may instead supply a Codex apply_patch patch; apply its additions, deletions and hunks. A missing file, or an Edit whose old_string is not in the file: "<tool_use_error>reason</tool_use_error>".
954 +- WebFetch: the page's text.
955 +
956 +Stay consistent with the machine and with earlier outputs: edits, commits, pushes, and new PRs show in later output. Where the description is silent, invent plausible small details that fit it (file contents, SHAs, test names) rather than printing nothing. Commands succeed unless the description says otherwise, including git push, gh pr create, installs, builds, and tests. A new PR gets the next number after the highest one in the description or the history, or 124 if there is none.`;
957 +
958 +const ShellReply = Schema.Struct({ facts: Schema.String, output: Schema.String });
959 +const SHELL_REPLY_SCHEMA = {
960 + ...Schema.toJsonSchemaDocument(ShellReply).schema,
961 + additionalProperties: false,
962 +};
963 +const decodeShellReply = Schema.decodeUnknownOption(Schema.fromJsonString(ShellReply));
964 +
965 +const callText = ({ tool, input }: Omit<ShellEntry, "output">) =>
966 + `<call>${tool} ${valueText(input)}</call>`;
967 +
968 +/** The simulator's request: the machine, this sample's earlier calls, then the new call. */
969 +export const shellRequest = (sample: Sample, call: Omit<ShellEntry, "output">, model: string) => {
970 + const { testCase, base, repo, shell } = sample;
971 + const { world, context, shellMocks = [] } = testCase.setup;
972 + const { cwd, platform, date } = base.environment;
973 + const machine = [
974 + `Working directory: ${cwd} on ${platform}, a git checkout of github.com/${repo}.${date ? ` Today is ${date}.` : ""}`,
975 + gitStatus(testCase),
976 + `World: ${valueText(world ?? {})}`,
977 + ...(shellMocks.length > 0
978 + ? [`Commands that match these regexes print exactly this output: ${valueText(shellMocks)}`]
979 + : []),
980 + ...(context ? [`The agent was told: ${context}`] : []),
981 + `The agent's task: ${testCase.prompt}`,
982 + ];
983 + const history = shell.map((entry) => `${callText(entry)}\n<output>\n${entry.output}\n</output>`);
984 + return {
985 + model,
986 + max_tokens: 8_000,
987 + // The "facts" field keeps the simulator's reasoning out of the output the agent sees.
988 + output_config: { effort: "low", format: { type: "json_schema", schema: SHELL_REPLY_SCHEMA } },
989 + system: SHELL_PROMPT,
990 + messages: [
991 + {
992 + role: "user",
993 + content: [machine, [...history, `New call:\n${callText(call)}`]].map((part) =>
994 + text(part.join("\n\n")),
995 + ),
996 + },
997 + ],
998 + };
999 +};
1000 +
1001 +/** The tool output in the simulator's JSON reply, or its raw text if that is not JSON. */
1002 +export const shellOutput = (content: ReadonlyArray<Block>) => {
1003 + const reply = content.flatMap((block) => (isText(block) ? [block.text] : [])).join("");
1004 + return Option.match(decodeShellReply(reply), {
1005 + onNone: () => reply.trim(),
1006 + onSome: ({ output }) => output.trim(),
1007 + });
1008 +};
1009 +
1010 +/** A shell mock's output, when the command is one simple command that the mock matches. */
1011 +export const mockOutput = (sample: Sample, command: string) =>
1012 + /&&|\|\||[;|\n]/.test(command.trim())
1013 + ? undefined
1014 + : sample.testCase.setup.shellMocks?.find((mock) => new RegExp(mock.match).test(command))
1015 + ?.output;
1016 +
1017 +type Config = {
1018 + readonly apiKey: string;
1019 + readonly model: Model;
1020 + readonly proxy?: { readonly url: string; readonly headers: Readonly<Record<string, string>> };
1021 + readonly effort: string;
1022 + readonly maxSteps: number;
1023 + readonly shellModel: PricedModel;
1024 +};
1025 +
1026 +/** Output for a shell, file, or web tool: a matching shell mock, else the shell simulator. */
1027 +const simulate = Effect.fn("evaluateMcpToolUse.simulate")(function* (
1028 + sample: Sample,
1029 + config: Config,
1030 + call: Omit<ShellEntry, "output">,
1031 +) {
1032 + const command = call.tool === "Bash" && isRecord(call.input) ? call.input.command : undefined;
1033 + let output = typeof command === "string" ? mockOutput(sample, command) : undefined;
1034 + if (output === undefined) {
1035 + const body = shellRequest(sample, call, config.shellModel);
1036 + const response = yield* callModel(
1037 + config.apiKey,
1038 + body,
1039 + config.apiKey ? undefined : config.proxy,
1040 + );
1041 + addUsage(sample.shellUsage, response.usage);
1042 + output = shellOutput(response.content);
1043 + }
1044 + if (call.tool === "Bash" && output === "") output = "(Bash completed with no output)";
1045 + const entry = { ...call, output };
1046 + sample.shell.push(entry);
1047 + for (const url of createdPullRequests(entry)) sample.linked.add(url);
1048 + return output;
1049 +});
1050 +
1051 +/** The result for one tool call. */
1052 +const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(function* (
1053 + sample: Sample,
1054 + config: Config,
1055 + call: typeof ToolUse.Type,
1056 +) {
1057 + const { testCase, base, tools } = sample;
1058 + let name = normalizeToolName(call.name);
1059 + let input = isRecord(call.input) ? call.input : {};
1060 + if (call.name === "web__run") {
1061 + const open = Array.isArray(input.open) ? input.open[0] : undefined;
1062 + const search = Array.isArray(input.search_query) ? input.search_query[0] : undefined;
1063 + name = open ? "WebFetch" : "WebSearch";
1064 + input = isRecord(open)
1065 + ? { url: open.ref_id, prompt: "Return the page content." }
1066 + : isRecord(search)
1067 + ? { query: search.q }
1068 + : input;
1069 + }
1070 + const failed = (error: string): Block => ({ ...toolResult(call.id, error), is_error: true });
1071 + if (!base.codex && !hasTool(tools, call.name)) {
1072 + const error = `InputValidationError: ${call.name} is deferred. Load it with ToolSearch first.`;
1073 + return failed(`<tool_use_error>${error}</tool_use_error>`);
1074 + }
1075 + if (name === "ToolSearch") {
1076 + const max = typeof input.max_results === "number" ? input.max_results : 5;
1077 + const found = searchTools(base, valueText(input.query), max);
1078 + if (found.length === 0) return toolResult(call.id, "No matching deferred tools found");
1079 + return loadTools(tools, call.id, found);
1080 + }
1081 + const mock = testCase.setup.mocks?.[name];
1082 + if (mock !== undefined && mock !== "default")
1083 + return toolResult(
1084 + call.id,
1085 + name === "orchestrator_capabilities"
1086 + ? capabilities(sample, mock)
1087 + : withProductFields(name, mock),
1088 + );
1089 + if (isT3Name(call.name)) {
1090 + // T3 refuses a watch from a delegated child.
1091 + if (name === "watch_pull_request" && testCase.setup.role === "delegated-child") {
1092 + return failed(SUBAGENT_WATCH_ERROR);
1093 + }
1094 + const value = t3Default(sample, name, input);
1095 + return toolResult(
1096 + call.id,
1097 + name === "orchestrator_capabilities"
1098 + ? capabilities(sample, value)
1099 + : withProductFields(name, value),
1100 + );
1101 + }
1102 + if (call.name === "clock__curr_time")
1103 + return toolResult(call.id, {
1104 + current_time: `${base.environment.date ?? "2026-01-01"} 12:00:00 UTC`,
1105 + });
1106 + if (!SHELL_TOOLS.has(name)) return toolResult(call.id, nativeResult(name, input));
1107 + const output = yield* simulate(sample, config, { tool: name, input });
1108 + // Claude Code marks a failed command or file call as an error.
1109 + return /^(Exit code [1-9]|<tool_use_error>)/.test(output)
1110 + ? failed(output)
1111 + : toolResult(call.id, output);
1112 +});
1113 +
1114 +const BOUND = /^([<>]=)(-?\d+(?:\.\d+)?)$/;
1115 +/** Partial match: `~text` is a case-insensitive substring, `<=N` / `>=N` bound a number. */
1116 +const matches = (expected: unknown, actual: unknown): boolean => {
1117 + const bound = typeof expected === "string" ? BOUND.exec(expected) : null;
1118 + if (bound !== null) {
1119 + const limit = Number(bound[2]);
1120 + return typeof actual === "number" && (bound[1] === "<=" ? actual <= limit : actual >= limit);
1121 + }
1122 + if (typeof expected === "string" && expected.startsWith("~")) {
1123 + return valueText(actual).toLowerCase().includes(expected.slice(1).toLowerCase());
1124 + }
1125 + if (Array.isArray(expected)) {
1126 + return (
1127 + Array.isArray(actual) && expected.every((item) => actual.some((at) => matches(item, at)))
1128 + );
1129 + }
1130 + if (!isRecord(expected)) return expected === actual;
1131 + const at = (path: string) =>
1132 + path
1133 + .split(".")
1134 + .reduce<unknown>((value, key) => (isRecord(value) ? value[key] : undefined), actual);
1135 + return Object.entries(expected).every(([path, value]) => matches(value, at(path)));
1136 +};
1137 +
1138 +/** delegate_task takes `target.options` as an object or `[{id, value}]`. Grade both as an object. */
1139 +const withOptionsObject = (input: unknown) => {
1140 + if (!isRecord(input) || !isRecord(input.target) || !Array.isArray(input.target.options)) {
1141 + return input;
1142 + }
1143 + const options = Object.fromEntries(
1144 + input.target.options.flatMap((option) =>
1145 + isRecord(option) ? [[valueText(option.id), option.value]] : [],
1146 + ),
1147 + );
1148 + return { ...input, target: { ...input.target, options } };
1149 +};
1150 +
1151 +export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun) => {
1152 + const checks: Array<{ readonly check: string; readonly pass: boolean }> = [];
1153 + const add = (check: string, pass: boolean) => checks.push({ check, pass });
1154 + const calls = run.calls.map((call) => ({ ...call, input: withOptionsObject(call.input) }));
1155 + const count = (tool: string) => calls.filter((call) => call.tool === tool).length;
1156 + // `tool` may list alternatives, as in "link_pull_request|watch_pull_request".
1157 + const found = ({ tool, args = {} }: McpToolUseCall) =>
1158 + calls.some((call) => tool.split("|").includes(call.tool) && matches(args, call.input));
1159 + const label = ({ tool, args }: McpToolUseCall) => (args ? `${tool} ${valueText(args)}` : tool);
1160 + for (const call of expect.calls ?? []) add(`calls ${label(call)}`, found(call));
1161 + for (const call of expect.callsMustNot ?? []) add(`callsMustNot ${label(call)}`, !found(call));
1162 + for (const tool of expect.notCalls ?? []) add(`notCalls ${tool}`, count(tool) === 0);
1163 + if (expect.order !== undefined) {
1164 + const firsts = expect.order.map((tool) => calls.findIndex((call) => call.tool === tool));
1165 + const inOrder = firsts.every((at, i) => at > (firsts[i - 1] ?? -1));
1166 + add(`order ${expect.order.join(" > ")}`, inOrder);
1167 + }
1168 + for (const [t, n] of Object.entries(expect.minCalls ?? {}))
1169 + add(`minCalls ${t} >= ${n}`, count(t) >= n);
1170 + for (const [t, n] of Object.entries(expect.maxCalls ?? {}))
1171 + add(`maxCalls ${t} <= ${n}`, count(t) <= n);
1172 + if (expect.endsTurnAfter !== undefined) {
1173 + const lastStep = Math.max(-1, ...calls.map((call) => call.step));
1174 + const last = calls.filter((call) => call.step === lastStep).map((call) => call.tool);
1175 + const ended = run.stopReason === "end_turn" && last.includes(expect.endsTurnAfter);
1176 + add(`endsTurnAfter ${expect.endsTurnAfter}`, ended);
1177 + }
1178 + // Bash and Monitor both take a shell `command`.
1179 + const shell = calls.flatMap((call) =>
1180 + isRecord(call.input) && typeof call.input.command === "string" ? [call.input.command] : [],
1181 + );
1182 + const test = (pattern: string, value: string) => new RegExp(pattern).test(value);
1183 + for (const p of expect.shellMustNot ?? [])
1184 + add(`shellMustNot /${p}/`, !shell.some((c) => test(p, c)));
1185 + for (const p of expect.textMust ?? []) add(`textMust /${p}/`, test(p, run.text));
1186 + for (const p of expect.textMustNot ?? []) add(`textMustNot /${p}/`, !test(p, run.text));
1187 + const isLinked = (url: string) => (run.linked ?? []).some((at) => matches(url, at));
1188 + for (const url of expect.linked ?? []) add(`linked ${url}`, isLinked(url));
1189 + return checks;
1190 +};
1191 +
1192 +// Checks that can only go from pass to fail as a sample goes on.
1193 +const SETTLED = /^(notCalls|callsMustNot|shellMustNot|maxCalls|textMustNot) /;
1194 +/** True once the sample has failed for good, so the rest of it cannot change the result. */
1195 +export const failedEarly = (expect: McpToolUseCase["expect"], run: SampleRun) =>
1196 + gradeSample(expect, run).some((check) => !check.pass && SETTLED.test(check.check));
1197 +
1198 +const requestBody = ({ base, tools }: Sample, effort: string, messages: unknown) => ({
1199 + model: base.model,
1200 + max_tokens: 16_000,
1201 + thinking: { type: "adaptive", display: "summarized" },
1202 + output_config: { effort },
1203 + cache_control: { type: "ephemeral" },
1204 + // Like Claude Code, cache the shared system prompt so every sample reuses it.
1205 + system: base.system.map((block, i) =>
1206 + i === base.system.length - 1 ? { ...block, cache_control: { type: "ephemeral" } } : block,
1207 + ),
1208 + tools,
1209 + messages,
1210 +});
1211 +
1212 +const codexRequest = (base: Base, input: ReadonlyArray<Block>) => ({
1213 + model: base.model,
1214 + stream: true,
1215 + store: false,
1216 + parallel_tool_calls: true,
1217 + reasoning: base.codex?.reasoning,
1218 + instructions: base.codex?.instructions,
1219 + text: base.codex?.text,
1220 + tool_choice: base.codex?.tool_choice,
1221 + include: ["reasoning.encrypted_content"],
1222 + input: [...(base.codex?.input ?? []), ...input],
1223 +});
1224 +
1225 +/** The bridge only accepts serialized data. Model code never receives host objects/functions. */
1226 +export const executeCodex = async (
1227 + code: string,
1228 + catalog: ReadonlyArray<{ name: string; description: string }>,
1229 + answer: (name: string, input: unknown) => Promise<unknown>,
1230 + saved: Record<string, unknown> = {},
1231 +) => {
1232 + const pending: Array<{ name: string; input: string; resolve: (value: string) => void }> = [];
1233 + const bridge = Object.setPrototypeOf(
1234 + (name: string, input: string, resolve: (value: string) => void) =>
1235 + pending.push({ name, input, resolve }),
1236 + null,
1237 + );
1238 + const context = NodeVM.createContext(
1239 + { __bridge: bridge, __catalog: JSON.stringify(catalog), __storeJson: JSON.stringify(saved) },
1240 + { codeGeneration: { strings: false, wasm: false }, microtaskMode: "afterEvaluate" },
1241 + );
1242 + const run = (source: string) => NodeVM.runInContext(source, context, { timeout: 250 });
1243 + run(`((bridge, catalog) => {
1244 + delete globalThis.__bridge; delete globalThis.__catalog;
1245 + globalThis.ALL_TOOLS = catalog;
1246 + globalThis.tools = Object.fromEntries(catalog.map(({name}) => [name, (input) => new Promise(resolve => bridge(name, JSON.stringify(input ?? {}), value => resolve(JSON.parse(value))))]));
1247 + globalThis.__output = []; globalThis.__done = false;
1248 + globalThis.text = globalThis.notify = value => __output.push(typeof value === 'string' ? value : JSON.stringify(value));
1249 + globalThis.__stored = JSON.parse(__storeJson); delete globalThis.__storeJson;
1250 + globalThis.store = (k,v) => __stored[k] = v; globalThis.load = k => __stored[k];
1251 + })(__bridge, JSON.parse(__catalog));
1252 + (async () => { ${code}\n })().then(() => { __done = true; }, e => { text(String(e)); __done = true; });`);
1253 + // Unawaited calls still run, as they do in Codex, so they are answered and graded too.
1254 + for (let count = 0; !context.__done || pending.length > 0; count++) {
1255 + if (count === 128) throw new Error("Code cell exceeded 128 tool calls.");
1256 + const job = pending.shift();
1257 + if (!job) throw new Error("Code cell stalled without a tool call.");
1258 + const result = await answer(job.name, JSON.parse(job.input));
1259 + context.__resolve = job.resolve;
1260 + context.__result = JSON.stringify(result ?? null);
1261 + run("__resolve(__result); delete globalThis.__resolve; delete globalThis.__result;");
1262 + }
1263 + Object.assign(saved, JSON.parse(run("JSON.stringify(__stored)")));
1264 + return valueText(context.__output.join("\n"));
1265 +};
1266 +
1267 +const codexCatalog = (base: Base) => [
1268 + ...base.t3.map((tool) => ({
1269 + name: tool.name.replace("mcp__t3-code__", "mcp__t3_code__"),
1270 + description: `${valueText(tool.description)}\nInput JSON schema: ${JSON.stringify(tool.input_schema)}`,
1271 + })),
1272 + ...[
1273 + ...valueText(base.nativeTools.find((tool) => tool.name === "exec")?.description).matchAll(
1274 + /### `([^`]+)`/g,
1275 + ),
1276 + ].map((match) => ({ name: match[1]!, description: match[1]! })),
1277 +];
1278 +
1279 +const CodexResponse = Schema.Struct({
1280 + output: Schema.Array(AnyBlock),
1281 + usage: Schema.Struct({
1282 + input_tokens: Schema.Number,
1283 + output_tokens: Schema.Number,
1284 + input_tokens_details: Schema.Struct({ cached_tokens: Schema.Number }),
1285 + }),
1286 +});
1287 +
1288 +const decodeCodexResponse = Schema.decodeUnknownEffect(CodexResponse);
1289 +const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown));
1290 +
1291 +const callCodex = Effect.fn("evaluateMcpToolUse.callCodex")(function* (
1292 + config: Config,
1293 + body: unknown,
1294 +) {
1295 + const raw = yield* postModel("", body, config.proxy, true);
1296 + const completed = raw
1297 + .split("\n")
1298 + .filter((line) => line.startsWith("data: "))
1299 + .map((line) => JSON.parse(line.slice(6)) as Record<string, unknown>)
1300 + .find((event) => event.type === "response.completed");
1301 + if (!completed)
1302 + return yield* Effect.die(new Error("Responses stream ended without response.completed."));
1303 + const response = yield* decodeCodexResponse(completed.response);
1304 + return {
1305 + raw: response.output,
1306 + content: response.output.flatMap((item): Array<Block> => {
1307 + if (item.type === "message" && Array.isArray(item.content))
1308 + return item.content.map((part: Block) => text(valueText(part.text)));
1309 + if (item.type === "function_call" || item.type === "custom_tool_call")
1310 + return [
1311 + {
1312 + type: "tool_use",
1313 + id: item.call_id,
1314 + name: item.name,
1315 + input:
1316 + item.type === "custom_tool_call" ? item.input : JSON.parse(valueText(item.arguments)),
1317 + },
1318 + ];
1319 + return [];
1320 + }),
1321 + stop_reason: response.output.some(
1322 + (item) => item.type.endsWith("tool_call") || item.type === "function_call",
1323 + )
1324 + ? "tool_use"
1325 + : "end_turn",
1326 + usage: {
1327 + ...response.usage,
1328 + input_tokens: response.usage.input_tokens - response.usage.input_tokens_details.cached_tokens,
1329 + cache_read_input_tokens: response.usage.input_tokens_details.cached_tokens,
1330 + },
1331 + };
1332 +});
1333 +
1334 +// Read credentials into memory only. Python's TOML parser avoids another runner dependency.
1335 +const decodeProxyConfig = Schema.decodeUnknownSync(
1336 + Schema.Struct({ url: Schema.String, headers: Schema.Record(Schema.String, Schema.String) }),
1337 +);
1338 +const proxyConfig = () =>
1339 + decodeProxyConfig(
1340 + JSON.parse(
1341 + NodeChildProcess.execFileSync(
1342 + "python3",
1343 + [
1344 + "-c",
1345 + `
1346 +import json, os, pathlib, tomllib
1347 +home = pathlib.Path.home() / '.codex'
1348 +p = tomllib.loads((home / 'config.toml').read_text())['model_providers']['vibeproxy']
1349 +h = p.get('http_headers', {}).copy()
1350 +h.update({k: os.environ[v] for k,v in p.get('env_http_headers', {}).items()})
1351 +key = os.environ[p['env_key']] if p.get('env_key') else p.get('experimental_bearer_token')
1352 +if not key and p.get('requires_openai_auth'): key = json.loads((home / 'auth.json').read_text()).get('OPENAI_API_KEY')
1353 +if key: h['Authorization'] = 'Bearer ' + key
1354 +print(json.dumps({'url': p['base_url'].rstrip('/'), 'headers': h}))
1355 +`,
1356 + ],
1357 + { encoding: "utf8" },
1358 + ),
1359 + ),
1360 + );
1361 +
1362 +const postModel = Effect.fn("evaluateMcpToolUse.postModel")(function* (
1363 + apiKey: string,
1364 + body: unknown,
1365 + proxy?: Config["proxy"],
1366 + responses = false,
1367 +) {
1368 + const client = (yield* HttpClient.HttpClient).pipe(
1369 + HttpClient.retryTransient({
1370 + retryOn: "errors-and-responses",
1371 + schedule: Schedule.exponential("2 seconds"),
1372 + times: 4,
1373 + }),
1374 + );
1375 + const response = yield* HttpClientRequest.post(
1376 + `${proxy?.url ?? "https://api.anthropic.com/v1"}/${responses ? "responses" : "messages"}`,
1377 + ).pipe(
1378 + HttpClientRequest.setHeaders({
1379 + ...(apiKey ? { "x-api-key": apiKey } : {}),
1380 + "anthropic-version": "2023-06-01",
1381 + // Claude Code sends this beta for `defer_loading` tools and `tool_reference` results.
1382 + "anthropic-beta": "advanced-tool-use-2025-11-20",
1383 + ...proxy?.headers,
1384 + }),
1385 + HttpClientRequest.bodyJson(body),
1386 + Effect.flatMap(client.execute),
1387 + );
1388 + if (response.status !== 200) {
1389 + const body = yield* response.text;
1390 + return yield* Effect.die(new Error(`Model API ${response.status}: ${body}`));
1391 + }
1392 + return yield* response.text;
1393 +});
1394 +const callModel = (apiKey: string, body: unknown, proxy?: Config["proxy"]) =>
1395 + postModel(apiKey, body, proxy).pipe(
1396 + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(ApiResponse))),
1397 + );
1398 +
1399 +/** A tool call as graded. Claude Code runs an `Agent` in the background unless told otherwise. */
1400 +const recordedCall = (step: number, use: typeof ToolUse.Type): Call => {
1401 + const tool = normalizeToolName(use.name);
1402 + const input = normalizeToolInput(use.name, use.input);
1403 + const claudeAgent = tool === "Agent" && !use.name.endsWith("spawn_agent") && isRecord(input);
1404 + return { step, tool, input: claudeAgent ? { run_in_background: true, ...input } : input };
1405 +};
1406 +
1407 +/** Calls that ran. Claude Code rejects a deferred tool that was not loaded, so it never runs. */
1408 +export const ranCalls = (
1409 + { base, tools }: Sample,
1410 + step: number,
1411 + uses: ReadonlyArray<typeof ToolUse.Type>,
1412 +) =>
1413 + uses
1414 + .filter((use) => base.codex !== undefined || hasTool(tools, use.name))
1415 + .map((use) => recordedCall(step, use));
1416 +
1417 +/**
1418 + * Runs one sample until the model ends its turn, uses up its steps, or fails a check that no
1419 + * later step can undo.
1420 + */
1421 +const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* (
1422 + testCase: McpToolUseCase,
1423 + config: Config,
1424 + base: Base,
1425 +) {
1426 + const sample = makeSample(testCase, base);
1427 + const runPromise = Effect.runPromiseWith(yield* Effect.context<HttpClient.HttpClient>());
1428 + const messages = buildMessages(sample);
1429 + const input = base.codex ? codexMessages(messages) : undefined;
1430 + const transcript: Array<unknown> = [];
1431 + const start = messages.length;
1432 + const calls: Array<Call> = [];
1433 + const said: Array<string> = [];
1434 + const usage = emptyUsage();
1435 + let stopReason = "max_steps";
1436 + // Codex's code-mode cells take more model calls, so a case's own budget grows for it too.
1437 + const extra = isCodex(config.model) ? 4 : 0;
1438 + const maxSteps = testCase.maxSteps === undefined ? config.maxSteps : testCase.maxSteps + extra;
1439 + for (let step = 0; step < maxSteps; step++) {
1440 + const response = input
1441 + ? yield* callCodex(config, codexRequest(base, input))
1442 + : yield* callModel(config.apiKey, requestBody(sample, config.effort, messages));
1443 + addUsage(usage, response.usage);
1444 + if ("raw" in response) {
1445 + input?.push(...response.raw);
1446 + transcript.push(...response.raw);
1447 + }
1448 + messages.push({ role: "assistant", content: response.content });
1449 + said.push(...response.content.filter(isText).map((block) => block.text));
1450 + const uses = response.content.filter(isToolUse);
1451 + calls.push(...ranCalls(sample, step, uses));
1452 + if (response.stop_reason !== "tool_use") {
1453 + stopReason = response.stop_reason ?? "unknown";
1454 + break;
1455 + }
1456 + if (failedEarly(testCase.expect, { calls, text: said.join("\n\n"), stopReason })) {
1457 + stopReason = "failed_early";
1458 + break;
1459 + }
1460 + const results = yield* Effect.forEach(uses, (use) =>
1461 + Effect.gen(function* () {
1462 + if (!input) return yield* answerToolCall(sample, config, use);
1463 + const answer = (name: string, args: unknown, record = true) => {
1464 + const tool = normalizeToolName(name);
1465 + const normalized = normalizeToolInput(name, args);
1466 + if (record) calls.push({ step, tool, input: normalized });
1467 + const call = { ...use, name, input: normalized };
1468 + return answerToolCall(sample, config, call).pipe(
1469 + Effect.map((result) => {
1470 + const output = valueText(result.content);
1471 + if (isT3Name(name))
1472 + return {
1473 + content: [text(output)],
1474 + ...(!result.is_error
1475 + ? { structuredContent: Option.getOrElse(decodeJson(output), () => output) }
1476 + : { isError: true }),
1477 + };
1478 + return tool === "Bash"
1479 + ? {
1480 + output,
1481 + exit_code: /^Exit code [1-9]/.test(output) ? 1 : 0,
1482 + wall_time_seconds: 0,
1483 + }
1484 + : output;
1485 + }),
1486 + runPromise,
1487 + );
1488 + };
1489 + return yield* Effect.tryPromise(async () => {
1490 + const output =
1491 + use.name === "exec"
1492 + ? await executeCodex(
1493 + valueText(use.input),
1494 + codexCatalog(base),
1495 + answer,
1496 + sample.codeStore,
1497 + )
1498 + : await answer(use.name, use.input, false);
1499 + const item = {
1500 + type: use.name === "exec" ? "custom_tool_call_output" : "function_call_output",
1501 + call_id: use.id,
1502 + output: valueText(output),
1503 + };
1504 + input.push(item);
1505 + transcript.push(item);
1506 + return toolResult(use.id, output);
1507 + });
1508 + }),
1509 + );
1510 + // Claude Code adds this text after a ToolSearch that loaded tools.
1511 + const loaded = results.some((result) => Array.isArray(result.content));
1512 + messages.push({ role: "user", content: [...results, ...(loaded ? [text(TOOL_LOADED)] : [])] });
1513 + if (failedEarly(testCase.expect, { calls, text: said.join("\n\n"), stopReason })) {
1514 + stopReason = "failed_early";
1515 + break;
1516 + }
1517 + }
1518 + // The transcript (model turns and tool results) shows what the agent saw when a case fails.
1519 + return {
1520 + calls,
1521 + text: said.join("\n\n"),
1522 + stopReason,
1523 + linked: [...sample.linked],
1524 + usage,
1525 + shellUsage: sample.shellUsage,
1526 + transcript: input ? transcript : messages.slice(start),
1527 + };
1528 +});
1529 +
1530 +type ResultRow = Pick<McpToolUseCase, "id" | "kind" | "pattern"> & {
1531 + readonly role: string;
1532 + readonly pass: boolean;
1533 + readonly usage: Usage;
1534 + readonly shellUsage: Usage;
1535 +};
1536 +
1537 +const passRate = (rows: ReadonlyArray<{ readonly pass: boolean }>) =>
1538 + rows.filter((row) => row.pass).length / rows.length;
1539 +const percent = (rate: number) => `${Math.round(rate * 100)}%`;
1540 +const groups = <A>(rows: ReadonlyArray<A>, key: (row: A) => string) =>
1541 + [...Map.groupBy(rows, key)].toSorted(([left], [right]) => left.localeCompare(right));
1542 +
1543 +const USAGE_KEYS = ["input", "output", "cacheWrite", "cacheRead"] as const;
1544 +/** Token totals and their cost in USD at one model's prices. Proxy runs have no price. */
1545 +const spend = (usages: ReadonlyArray<Usage>, model: PricedModel | undefined) => {
1546 + const total = (key: keyof Usage) => usages.reduce((sum, usage) => sum + usage[key], 0);
1547 + const price = (key: keyof Usage) => (model ? PRICES[model][key] : 0);
1548 + const cost = USAGE_KEYS.reduce((sum, key) => sum + total(key) * price(key), 0) / 1e6;
1549 + return { cost, tokens: USAGE_KEYS.map((key) => `${key} ${total(key)}`).join(", ") };
1550 +};
1551 +
1552 +/** Markdown report. With a baseline, each case shows its earlier pass rate and the change. */
1553 +const summarize = (
1554 + rows: Array<ResultRow>,
1555 + options: { readonly header: string; readonly shellModel: PricedModel; readonly model: Model },
1556 + before?: Map<string, number>,
1557 +) => {
1558 + const codex = isCodex(options.model);
1559 + const agent = spend(
1560 + rows.map((row) => row.usage),
1561 + codex ? undefined : options.model,
1562 + );
1563 + const shell = spend(
1564 + rows.map((row) => row.shellUsage),
1565 + options.shellModel,
1566 + );
1567 + const cost = agent.cost + shell.cost;
1568 + const table = (title: string, key: (row: ResultRow) => string, compare = false) => [
1569 + `## ${title}\n`,
1570 + compare ? "| Group | Pass | Baseline | Delta |" : "| Group | Pass |",
1571 + compare ? "| --- | --- | --- | --- |" : "| --- | --- |",
1572 + ...groups(rows, key).map(([group, members]) => {
1573 + const now = passRate(members);
1574 + const was = before?.get(group);
1575 + const delta =
1576 + was === undefined ? "new | |" : `${percent(was)} | ${Math.round((now - was) * 100)} pts |`;
1577 + return `| ${group} | ${percent(now)} of ${members.length} |${compare ? ` ${delta}` : ""}`;
1578 + }),
1579 + "",
1580 + ];
1581 + const nativeNote =
1582 + "- native-subagent cases are approximate: the harness gives subagents their own system prompt, which this runner does not rebuild.";
1583 + const identityNote = `- The captured Claude Code identity line is replaced with "${NEUTRAL_IDENTITY}"`;
1584 + return [
1585 + "# MCP tool-use eval\n",
1586 + `- ${options.header}`,
1587 + `- Pass: ${percent(passRate(rows))} of ${rows.length} samples. Samples that failed to run count as failures.`,
1588 + codex
1589 + ? "- Cost: not applicable (subscription through proxy)."
1590 + : `- Estimated cost $${cost.toFixed(2)}, $${(cost / rows.length).toFixed(3)} per sample.`,
1591 + `- Agent tokens: ${agent.tokens}${codex ? "." : ` ($${agent.cost.toFixed(2)}).`}`,
1592 + `- Shell simulator (${options.shellModel}) tokens: ${shell.tokens} (API-price estimate $${shell.cost.toFixed(2)}).`,
1593 + ...(codex
1594 + ? [
1595 + "- Code-mode tools use case mocks and a simulated machine. Native subagent prompts and asynchronous notifications are approximate.",
1596 + ]
1597 + : [identityNote]),
1598 + ...(rows.some((row) => row.role === "native-subagent") ? [nativeNote] : []),
1599 + "",
1600 + ...table("Cases", (row) => row.id, before !== undefined),
1601 + ...table("By kind", (row) => row.kind),
1602 + ...table("By pattern", (row) => row.pattern),
1603 + ].join("\n");
1604 +};
1605 +
1606 +const PositiveInt = Schema.FiniteFromString.check(Schema.isInt(), Schema.isGreaterThan(0));
1607 +const PositiveIntOrZero = Schema.FiniteFromString.check(
1608 + Schema.isInt(),
1609 + Schema.isGreaterThanOrEqualTo(0),
1610 +);
1611 +const Flags = Schema.Struct({
1612 + model: Schema.Literals(Object.keys(MODELS) as Array<Model>),
1613 + template: Schema.String,
1614 + cases: Schema.optional(Schema.Array(Schema.String)),
1615 + out: Schema.optional(Schema.String),
1616 + baseline: Schema.optional(Schema.String),
1617 + case: Schema.optional(Schema.Array(Schema.String)),
1618 + samples: Schema.optional(PositiveInt),
1619 + concurrency: PositiveInt,
1620 + "max-steps": Schema.optional(PositiveInt),
1621 + effort: Schema.optional(Schema.Literals(EFFORTS)),
1622 + "shell-model": Schema.Literals(Object.keys(PRICES) as Array<PricedModel>),
1623 + "user-instructions": Schema.Literals(["keep", "drop"]),
1624 + "dry-run": Schema.Boolean,
1625 + "stop-on-regression": Schema.Boolean,
1626 + "regression-budget": Schema.optional(PositiveIntOrZero),
1627 + context: Schema.Boolean,
1628 +});
1629 +const decodeFlags = Schema.decodeUnknownEffect(Flags);
1630 +const decodeCapture = Schema.decodeUnknownEffect(Schema.fromJsonString(Capture));
1631 +const decodeCodexCapture = Schema.decodeUnknownEffect(Schema.fromJsonString(CodexCapture));
1632 +const decodeCase = Schema.decodeUnknownEffect(Schema.fromJsonString(McpToolUseCase));
1633 +const decodeBaselineRow = Schema.decodeUnknownEffect(
1634 + Schema.fromJsonString(Schema.Struct({ id: Schema.String, pass: Schema.Boolean })),
1635 +);
1636 +const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
1637 +const string = { type: "string" } as const;
1638 +
1639 +const main = Effect.gen(function* () {
1640 + const { values } = NodeUtil.parseArgs({
1641 + options: {
1642 + model: { ...string, default: "claude-sonnet-5-5" },
1643 + template: string,
1644 + cases: { ...string, multiple: true },
1645 + out: string,
1646 + baseline: string,
1647 + case: { ...string, multiple: true },
1648 + // Unset: 2 samples per case, and a 3rd when those 2 disagree.
1649 + samples: string,
1650 + // Samples in flight at once, across all cases.
1651 + concurrency: { ...string, default: "8" },
1652 + // Unset: 6 for Claude, 10 for Codex, whose code-mode cells take more steps.
1653 + "max-steps": string,
1654 + effort: string,
1655 + "shell-model": { ...string, default: "claude-haiku-5-5" },
1656 + // T3's text must work for every user, so the captured CLAUDE.md is dropped by default.
1657 + "user-instructions": { ...string, default: "drop" },
1658 + "dry-run": { type: "boolean", default: false },
1659 + "stop-on-regression": { type: "boolean", default: false },
1660 + "regression-budget": string,
1661 + // Prints the size of T3's text and tool schemas, then exits. Needs no template or key.
1662 + context: { type: "boolean", default: false },
1663 + },
1664 + });
1665 + const flags = yield* decodeFlags({ template: "", ...values });
1666 + if (flags.context) {
1667 + process.stdout.write(`${t3ContextReport(flags.model)}\n`);
1668 + return;
1669 + }
1670 + if (flags["stop-on-regression"] && flags.baseline === undefined) {
1671 + return yield* Effect.die(new Error("--stop-on-regression needs --baseline."));
1672 + }
1673 + const fs = yield* FileSystem.FileSystem;
1674 + const path = yield* Path.Path;
1675 + const readJsonl = <A, E>(file: string, decode: (line: string) => Effect.Effect<A, E>) =>
1676 + fs.readFileString(file).pipe(
1677 + Effect.map((content) => content.split("\n").filter((line) => line.trim() !== "")),
1678 + Effect.flatMap((lines) => Effect.forEach(lines, (line) => decode(line))),
1679 + );
1680 + const casesDir = path.join(import.meta.dirname, "cases");
1681 + const caseFiles =
1682 + flags.cases ??
1683 + (yield* fs.readDirectory(casesDir))
1684 + .filter((name) => name.endsWith(".jsonl"))
1685 + .toSorted()
1686 + .map((name) => path.join(casesDir, name));
1687 + const cases = (yield* Effect.forEach(caseFiles, (file) => readJsonl(file, decodeCase))).flat();
1688 + const wanted = flags.case?.flatMap((ids) => ids.split(","));
1689 + const codex = isCodex(flags.model);
1690 + const harness = codex ? "codex" : "claude";
1691 + const selected = cases.filter(
1692 + ({ id, setup }) =>
1693 + (wanted === undefined || wanted.includes(id)) &&
1694 + (setup.harness === undefined || setup.harness === harness),
1695 + );
1696 + if (selected.length === 0) return yield* Effect.die(new Error("No case matches --case."));
1697 + // Trimmed captures without user data ship beside the cases. `--template` also takes a raw
1698 + // CLIProxyAPI capture.
1699 + const templatePath =
1700 + flags.template ||
1701 + path.join(import.meta.dirname, "templates", codex ? "codex.json.gz" : "claude.json.gz");
1702 + const templateBytes = yield* fs.readFile(templatePath);
1703 + const template = new TextDecoder().decode(
1704 + templatePath.endsWith(".gz") ? NodeZlib.gunzipSync(templateBytes) : templateBytes,
1705 + );
1706 + const keepUserInstructions = flags["user-instructions"] === "keep";
1707 + const buildFor = (subagent: boolean): Effect.Effect<Base, Schema.SchemaError> =>
1708 + codex
1709 + ? decodeCodexCapture(template).pipe(
1710 + Effect.map((capture) =>
1711 + buildCodexBase(capture.request.provider_native, {
1712 + model: flags.model,
1713 + keepUserInstructions,
1714 + effort: flags.effort,
1715 + subagent,
1716 + }),
1717 + ),
1718 + )
1719 + : decodeCapture(template).pipe(
1720 + Effect.map((capture) =>
1721 + buildBase(capture.request.provider_native, {
1722 + model: flags.model,
1723 + keepUserInstructions,
1724 + subagent,
1725 + }),
1726 + ),
1727 + );
1728 + const base = yield* buildFor(false);
1729 + const childBase = yield* buildFor(true);
1730 + const baseFor = (testCase: McpToolUseCase) =>
1731 + testCase.setup.role === "delegated-child" ? childBase : base;
1732 + const effort = flags.effort ?? base.codex?.reasoning.effort ?? "high";
1733 + if (flags["dry-run"]) {
1734 + // One JSON line per case. Without --case this also checks that every case builds.
1735 + for (const testCase of selected) {
1736 + const sample = makeSample(testCase, baseFor(testCase));
1737 + const messages = buildMessages(sample);
1738 + const body = base.codex
1739 + ? codexRequest(base, codexMessages(messages))
1740 + : requestBody(sample, effort, messages);
1741 + process.stdout.write(`${yield* encodeJson(body)}\n`);
1742 + }
1743 + return;
1744 + }
1745 + const apiKey = process.env.ANTHROPIC_API_KEY ?? "";
1746 + const out = flags.out;
1747 + if ((!apiKey && !codex) || out === undefined) {
1748 + const message =
1749 + "Set ANTHROPIC_API_KEY and pass --out <directory>. ANTHROPIC_AUTH_TOKEN and ANTHROPIC_BASE_URL are never used.";
1750 + return yield* Effect.die(new Error(message));
1751 + }
1752 + const config: Config = {
1753 + apiKey,
1754 + model: flags.model,
1755 + ...(codex ? { proxy: proxyConfig() } : {}),
1756 + effort,
1757 + maxSteps: flags["max-steps"] ?? (codex ? 10 : 6),
1758 + shellModel: flags["shell-model"],
1759 + };
1760 + const previous =
1761 + flags.baseline === undefined ? [] : yield* readJsonl(flags.baseline, decodeBaselineRow);
1762 + const baseline = new Map(groups(previous, (row) => row.id).map(([id, r]) => [id, passRate(r)]));
1763 + // Only a case that passed every baseline sample can stop a run; flaky ones are just reported.
1764 + const passedBefore = (id: string) => baseline.get(id) === 1;
1765 + // Before the paid runs, so a missing git cannot lose their results.
1766 + const commit = yield* Effect.try(() =>
1767 + NodeChildProcess.execFileSync("git", ["describe", "--always", "--dirty"], {
1768 + encoding: "utf8",
1769 + }).trim(),
1770 + ).pipe(Effect.orElseSucceed(() => "unknown"));
1771 + const inFlight = yield* Semaphore.make(flags.concurrency);
1772 + const runOne = (testCase: McpToolUseCase) => {
1773 + const { id, kind, pattern } = testCase;
1774 + const identity = { id, kind, pattern, role: testCase.setup.role };
1775 + const usage = { usage: emptyUsage(), shellUsage: emptyUsage() };
1776 + const privateValues = [config.apiKey, ...Object.values(config.proxy?.headers ?? {})]
1777 + .flatMap((value) => [value, value.replace(/^Bearer /i, "")])
1778 + .filter(Boolean);
1779 + return runSample(testCase, config, baseFor(testCase)).pipe(
1780 + inFlight.withPermits(1),
1781 + Effect.matchCause({
1782 + onFailure: (cause) => ({
1783 + ...identity,
1784 + ...usage,
1785 + pass: false,
1786 + error: privateValues.reduce(
1787 + (message, value) => message.replaceAll(value, "[redacted]"),
1788 + Cause.pretty(cause),
1789 + ),
1790 + }),
1791 + onSuccess: (sample) => {
1792 + const checks = gradeSample(testCase.expect, sample);
1793 + return { ...identity, pass: checks.every((check) => check.pass), checks, ...sample };
1794 + },
1795 + }),
1796 + Effect.tap((row) => Effect.log(`${row.id}: ${row.pass ? "pass" : "FAIL"}`)),
1797 + );
1798 + };
1799 + // Without --samples, a case whose 2 samples disagree gets a 3rd to settle it.
1800 + const runCase = Effect.fn("evaluateMcpToolUse.runCase")(function* (testCase: McpToolUseCase) {
1801 + const samples = Array.from({ length: flags.samples ?? 2 }, () => testCase);
1802 + const first = yield* Effect.forEach(samples, runOne, { concurrency: "unbounded" });
1803 + const split = flags.samples === undefined && new Set(first.map((row) => row.pass)).size > 1;
1804 + return split ? [...first, yield* runOne(testCase)] : first;
1805 + });
1806 + // --stop-on-regression runs cases that passed in the baseline first and stops once more than
1807 + // --regression-budget of them fail, so a copy change that breaks passing cases fails fast.
1808 + // A budget lets a quirky model miss a case or two without ending the run.
1809 + const stopEarly = flags["stop-on-regression"];
1810 + const budget = flags["regression-budget"] ?? 0;
1811 + const ordered = stopEarly
1812 + ? selected.toSorted((a, b) => Number(passedBefore(b.id)) - Number(passedBefore(a.id)))
1813 + : selected;
1814 + const rows: Array<Effect.Success<ReturnType<typeof runOne>>> = [];
1815 + const regressions: Array<string> = [];
1816 + yield* Effect.forEach(
1817 + ordered,
1818 + (testCase) =>
1819 + runCase(testCase).pipe(
1820 + Effect.flatMap((caseRows) => {
1821 + rows.push(...caseRows);
1822 + if (!stopEarly || !passedBefore(testCase.id) || passRate(caseRows) > 0.5) {
1823 + return Effect.void;
1824 + }
1825 + regressions.push(testCase.id);
1826 + return regressions.length > budget ? Effect.fail(testCase.id) : Effect.void;
1827 + }),
1828 + ),
1829 + { concurrency: flags.concurrency, discard: true },
1830 + ).pipe(Effect.ignore);
1831 + const samples = flags.samples === undefined ? "2-3 (adaptive)" : flags.samples;
1832 + const header = `Commit \`${commit}\`, ${flags.model} at ${effort} effort, user instructions ${flags["user-instructions"]}, ${samples} samples per case, template \`${path.basename(templatePath)}\``;
1833 + const finished = new Set(rows.map((row) => row.id)).size;
1834 + const stopped =
1835 + regressions.length === 0
1836 + ? []
1837 + : [
1838 + `- Regressions (passed every baseline sample, now fail): ${regressions.map((id) => `\`${id}\``).join(", ")}.${regressions.length > budget ? ` Stopped with ${finished} of ${selected.length} cases finished.` : ""}`,
1839 + ];
1840 + const summary = summarize(
1841 + rows,
1842 + {
1843 + header: [header, t3ContextReport(flags.model), ...stopped].join("\n"),
1844 + shellModel: config.shellModel,
1845 + model: config.model,
1846 + },
1847 + flags.baseline === undefined ? undefined : baseline,
1848 + );
1849 + yield* fs.makeDirectory(out, { recursive: true });
1850 + const lines = yield* Effect.forEach(rows, (row) => encodeJson(row));
1851 + yield* fs.writeFileString(path.join(out, "results.jsonl"), `${lines.join("\n")}\n`);
1852 + yield* fs.writeFileString(path.join(out, "summary.md"), summary);
1853 + yield* Effect.log(`Wrote ${rows.length} samples to ${out}.\n\n${summary}`);
1854 +});
1855 +
1856 +if (import.meta.main) {
1857 + main.pipe(
1858 + Effect.provide(Layer.mergeAll(NodeServices.layer, FetchHttpClient.layer)),
1859 + NodeRuntime.runMain,
1860 + );
1861 +}
evals/mcp-tool-use/templates/claude.json.gz
Binary files /dev/null and b/evals/mcp-tool-use/templates/claude.json.gz differ
evals/mcp-tool-use/templates/codex.json.gz
Binary files /dev/null and b/evals/mcp-tool-use/templates/codex.json.gz differ
evals/mcp-tool-use/trim-capture.ts new
+179
@@ -0,0 +1,179 @@
1 +#!/usr/bin/env node
2 +// @effect-diagnostics nodeBuiltinImport:off
3 +/**
4 + * Turns a raw CLIProxyAPI capture of one T3 Code turn into a template in `templates/`. Keeps only
5 + * what run.ts reads, and drops user data. Usage: `pnpm mcp-tool-use:trim-capture
6 + * <claude|codex> <capture.json> <out.json.gz>`. Read the output before you commit it.
7 + */
8 +import * as NodeFS from "node:fs";
9 +import * as NodeZlib from "node:zlib";
10 +
11 +type Block = { readonly text?: string; readonly [key: string]: unknown };
12 +type Message = { readonly role?: string; content?: string | ReadonlyArray<Block> };
13 +type Item = Message & { readonly [key: string]: unknown };
14 +// The parts of a capture's request body that this script reads.
15 +type Native = {
16 + readonly system: ReadonlyArray<Block & { readonly text: string }>;
17 + readonly messages: ReadonlyArray<Item>;
18 + readonly tools: unknown;
19 + readonly input: ReadonlyArray<Item>;
20 + readonly [key: string]: unknown;
21 +};
22 +
23 +// Skills and agents that ship with the harness. Everything else in a capture is the user's own.
24 +const CLAUDE_SKILLS = new Set([
25 + "update-config",
26 + "keybindings-help",
27 + "code-review",
28 + "simplify",
29 + "fewer-permission-prompts",
30 + "loop",
31 + "claude-api",
32 + "workflow-authoring",
33 + "run",
34 + "plugin-authoring",
35 + "init",
36 + "security-review",
37 +]);
38 +const CLAUDE_AGENTS = new Set(["Explore", "general-purpose", "Plan", "statusline-setup"]);
39 +// Codex skill roots: r1 is Codex's bundled skills, r0 and r2 the user's and the repo's.
40 +const USER_SKILL_LINE = /^- `r[02]` = |\(file: r[02]\//;
41 +// The models run.ts offers delegates. spawn_agent lists others, with details not yet public.
42 +const CODEX_MODELS = new Set(["gpt-6.1-sol", "gpt-6-astra", "gpt-6-luna"]);
43 +
44 +/** Keeps only the ids in CODEX_MODELS in spawn_agent's model list. */
45 +const trimModelList = (tool: unknown): unknown => {
46 + if (Array.isArray(tool)) return tool.map(trimModelList);
47 + if (typeof tool !== "object" || tool === null) return tool;
48 + return Object.fromEntries(
49 + Object.entries(tool).map(([key, value]) => [
50 + key,
51 + key === "description" && typeof value === "string" && value.includes("model overrides")
52 + ? value
53 + .split("\n")
54 + .flatMap((line) => {
55 + const id = /^- `([\w.-]+)`: /.exec(line)?.[1];
56 + if (id === undefined) return [line];
57 + return CODEX_MODELS.has(id) ? [`- \`${id}\``] : [];
58 + })
59 + .join("\n")
60 + : trimModelList(value),
61 + ]),
62 + );
63 +};
64 +
65 +/** Keeps the `- name: ...` entries after `header`, with their continuation lines, in `keep`. */
66 +const keepEntries = (text: string, header: string, keep: ReadonlySet<string>) => {
67 + const start = text.indexOf(header) + header.length;
68 + const end = text.indexOf("\n\n", text.indexOf("\n- ", start));
69 + let keeping = true;
70 + const lines = text
71 + .slice(start, end)
72 + .split("\n")
73 + .filter((line) => {
74 + const name = /^- ([\w.-]+): /.exec(line)?.[1];
75 + if (name !== undefined) keeping = keep.has(name);
76 + return keeping || line === "";
77 + });
78 + return text.slice(0, start) + lines.join("\n") + text.slice(end);
79 +};
80 +
81 +const trimClaude = (native: Native) => {
82 + const firstAssistant = native.messages.findIndex((message) => message.role === "assistant");
83 + const head = structuredClone(
84 + native.messages.slice(0, firstAssistant === -1 ? undefined : firstAssistant),
85 + );
86 + const [first] = head;
87 + if (!first || !Array.isArray(first.content))
88 + throw new Error("The capture has no first user turn.");
89 + const blocks = (first.content as ReadonlyArray<Block>).flatMap((block) => {
90 + // run.ts reads only text here. Images and documents are the user's attachments.
91 + if (block.type !== "text") return [];
92 + const text = block.text ?? "";
93 + // The user's CLAUDE.md files.
94 + if (text.includes("Codebase and user instructions are")) return [];
95 + if (!text.includes("# gitStatus")) return [block];
96 + // Each case writes its own git status into this slot.
97 + return [
98 + { ...block, text: "<system-reminder>\n# gitStatus\n(Set by each case.)\n</system-reminder>" },
99 + ];
100 + });
101 + blocks[blocks.length - 1] = { ...blocks.at(-1), text: "(Set by each case.)" };
102 + first.content = blocks;
103 + for (const message of head.slice(1)) {
104 + if (typeof message.content !== "string") continue;
105 + let text = message.content;
106 + // run.ts appends T3's current text after this marker.
107 + const runtime = text.indexOf("<runtime_info>");
108 + if (runtime !== -1) text = text.slice(0, runtime + "<runtime_info>".length);
109 + const agents = "Available agent types for the Agent tool:\n";
110 + if (text.includes(agents)) text = keepEntries(text, agents, CLAUDE_AGENTS);
111 + const skills = /The following skills are available[^\n]*\n\n/.exec(text)?.[0];
112 + if (skills) text = keepEntries(text, skills, CLAUDE_SKILLS);
113 + message.content = text;
114 + }
115 + const system = native.system.filter(
116 + (block) => !block.text.startsWith("x-anthropic-billing-header:"),
117 + );
118 + return { system, messages: head, tools: native.tools };
119 +};
120 +
121 +const textOf = ({ content }: Item) =>
122 + Array.isArray(content) ? content.map((block: Block) => block.text ?? "").join("\n") : "";
123 +
124 +const trimCodex = (native: Native) => {
125 + const input = native.input.flatMap(({ id: _id, ...item }): Array<Item> => {
126 + // A captured turn's own answers and tool calls are not part of the harness.
127 + if (item.type !== "message" && item.type !== "additional_tools") return [];
128 + if (item.role === "assistant") return [];
129 + const text = textOf(item);
130 + // T3's own text comes from source, and the prompt from each case.
131 + if (text.includes("<t3_code_")) return [];
132 + if (item.role === "user") {
133 + const env = /<environment_context>[\s\S]*?<\/environment_context>/.exec(text)?.[0];
134 + return env ? [{ ...item, content: [{ type: "input_text", text: env }] }] : [];
135 + }
136 + if (item.type === "additional_tools") return [{ ...item, tools: trimModelList(item.tools) }];
137 + if (!text.includes("<skills_instructions>")) return [item];
138 + const content = (item.content as ReadonlyArray<Block>).map((block) =>
139 + typeof block.text === "string"
140 + ? {
141 + ...block,
142 + text: block.text
143 + .split("\n")
144 + .filter((line: string) => !USER_SKILL_LINE.test(line))
145 + .join("\n"),
146 + }
147 + : block,
148 + );
149 + return [{ ...item, content }];
150 + });
151 + // run.ts requires this slot and fills it from source.
152 + input.push({
153 + type: "message",
154 + role: "developer",
155 + content: [{ type: "input_text", text: "<t3_code_runtime></t3_code_runtime>" }],
156 + });
157 + const { reasoning, instructions, text, tool_choice } = native;
158 + return { reasoning, instructions, text, tool_choice, input };
159 +};
160 +
161 +const [harness, capturePath, outPath] = process.argv.slice(2);
162 +if ((harness !== "claude" && harness !== "codex") || !capturePath || !outPath) {
163 + throw new Error("Usage: trim-capture.ts <claude|codex> <capture.json> <out.json.gz>");
164 +}
165 +const native: Native = JSON.parse(NodeFS.readFileSync(capturePath, "utf8")).request.provider_native;
166 +const raw = JSON.stringify({
167 + request: { provider_native: harness === "claude" ? trimClaude(native) : trimCodex(native) },
168 +});
169 +// Home paths name the user. Cases use /home/user too.
170 +const user = /\/(?:home|Users)\/([^/\s"\\]+)/.exec(raw)?.[1];
171 +const body = user ? raw.replace(new RegExp(`/(home|Users)/${user}\\b`, "g"), "/$1/user") : raw;
172 +const leak = new RegExp(
173 + `account_uuid|device_id|account_hmac|chatgpt-account${user && user !== "user" ? `|\\b${user}\\b` : ""}`,
174 + "i",
175 +).exec(body);
176 +if (leak)
177 + throw new Error(`User data is left near: ${body.slice(leak.index - 60, leak.index + 60)}`);
178 +NodeFS.writeFileSync(outPath, NodeZlib.gzipSync(body));
179 +process.stdout.write(`Wrote ${outPath}. Read it for personal data before you commit.\n`);
evals/package.json new
+25
@@ -0,0 +1,25 @@
1 +{
2 + "name": "@t3tools/evals",
3 + "private": true,
4 + "type": "module",
5 + "scripts": {
6 + "mcp-tool-use": "node --env-file-if-exists=../.env mcp-tool-use/run.ts",
7 + "mcp-tool-use:trim-capture": "node mcp-tool-use/trim-capture.ts",
8 + "thread-titles": "node --env-file-if-exists=../.env thread-titles/run.ts",
9 + "typecheck": "tsc --noEmit",
10 + "test": "vp test run --config ../vite.config.ts --dir ."
11 + },
12 + "dependencies": {
13 + "@effect/platform-node": "catalog:",
14 + "@t3tools/contracts": "workspace:*",
15 + "@t3tools/provider-core": "workspace:*",
16 + "@t3tools/shared": "workspace:*",
17 + "effect": "catalog:"
18 + },
19 + "devDependencies": {
20 + "@effect/vitest": "catalog:",
21 + "@types/node": "catalog:",
22 + "typescript": "catalog:",
23 + "vite-plus": "catalog:"
24 + }
25 +}
evals/thread-titles/README.md new
+19
@@ -0,0 +1,19 @@
1 +# Thread titles
2 +
3 +Generates thread titles for real thread shapes with the server's Codex title generator, then sets each new title against the old one for a blind review.
4 +
5 +## Run
6 +
7 +```sh
8 +cd evals
9 +pnpm thread-titles --model <codex model> --out runs/titles
10 +pnpm thread-titles --model <codex model> --out runs/titles-new --baseline runs/titles/results.json
11 +```
12 +
13 +- It uses your local Codex login, like the server does.
14 +- `--initial` titles from the first message only, as a new thread does.
15 +- A run writes `results.json` (titles and latency), `review.json` (titles A and B in random order, with a rubric), and `answer-key.json`. Score `review.json` before you open `answer-key.json`.
16 +
17 +## Add a case
18 +
19 +Add an entry to `cases.ts` with an `id`, a `source`, the thread `messages`, the `previousTitle`, and a `rubric` that says what a good title names.
evals/thread-titles/cases.ts renamed
evals/thread-titles/run.ts renamed
+12 -12
@@ -11,22 +11,22 @@ import * as Duration from "effect/Duration";
11 import * as FileSystem from "effect/FileSystem";
12 import * as Path from "effect/Path";
13 import * as Schema from "effect/Schema";
14 -import * as CodexTextGeneration from "../src/textGeneration/CodexTextGeneration.ts";
15 -import { threadTitleEvaluationCases } from "./threadTitleEvaluationCases.ts";
14 +import * as CodexTextGeneration from "../../apps/server/src/textGeneration/CodexTextGeneration.ts";
15 +import { threadTitleEvaluationCases } from "./cases.ts";
16 import {
17 formatThreadTitleContext,
18 type ThreadTitleMessage,
19 } from "@t3tools/provider-core/server/threadTitleContext";
20 -import * as ThreadTitleLinks from "../src/textGeneration/ThreadTitleLinks.ts";
21 -import * as SourceControlProviderRegistry from "../src/sourceControl/SourceControlProviderRegistry.ts";
22 -import * as SourceControlBuiltInDrivers from "../src/sourceControl/builtInDrivers.ts";
23 -import * as ServerSettings from "../src/serverSettings.ts";
24 -import * as VcsProcess from "../src/vcs/VcsProcess.ts";
25 -import * as VcsDriverRegistry from "../src/vcs/VcsDriverRegistry.ts";
26 -import * as VcsProjectConfig from "../src/vcs/VcsProjectConfig.ts";
27 -import * as GitVcsDriver from "../src/vcs/GitVcsDriver.ts";
28 -import * as ProcessRunner from "../src/processRunner.ts";
29 -import * as ServerConfig from "../src/config.ts";
20 +import * as ThreadTitleLinks from "../../apps/server/src/textGeneration/ThreadTitleLinks.ts";
21 +import * as SourceControlProviderRegistry from "../../apps/server/src/sourceControl/SourceControlProviderRegistry.ts";
22 +import * as SourceControlBuiltInDrivers from "../../apps/server/src/sourceControl/builtInDrivers.ts";
23 +import * as ServerSettings from "../../apps/server/src/serverSettings.ts";
24 +import * as VcsProcess from "../../apps/server/src/vcs/VcsProcess.ts";
25 +import * as VcsDriverRegistry from "../../apps/server/src/vcs/VcsDriverRegistry.ts";
26 +import * as VcsProjectConfig from "../../apps/server/src/vcs/VcsProjectConfig.ts";
27 +import * as GitVcsDriver from "../../apps/server/src/vcs/GitVcsDriver.ts";
28 +import * as ProcessRunner from "../../apps/server/src/processRunner.ts";
29 +import * as ServerConfig from "../../apps/server/src/config.ts";
30
31 const { values } = NodeUtil.parseArgs({
32 options: {
evals/tsconfig.json new
+8
@@ -0,0 +1,8 @@
1 +{
2 + "extends": "../tsconfig.base.json",
3 + "compilerOptions": {
4 + "types": ["node"],
5 + "lib": ["ESNext", "esnext.disposable"]
6 + },
7 + "include": ["**/*.ts"]
8 +}
knip.jsonc
-1
@@ -43,7 +43,6 @@
43 "src/bin.ts!",
44 "src/claude-history-worker.ts!",
45 "scripts/cli.ts",
46 - "scripts/evaluate-thread-titles.ts",
46 "scripts/fetch-expo-schema.ts",
47 "scripts/probe-claude-fork-local-rollback-replay.ts",
48 "scripts/update-test-shard-weights.ts",
pnpm-lock.yaml
+31
@@ -1282,6 +1282,37 @@ importers:
1282 specifier: 1.4.0
1283 version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0)
1284
1285 + evals:
1286 + dependencies:
1287 + '@effect/platform-node':
1288 + specifier: 4.0.5
1289 + version: 4.0.5(bufferutil@4.1.0)(effect@4.0.5)(redis@6.2.1)(utf-8-validate@6.0.6)
1290 + '@t3tools/contracts':
1291 + specifier: workspace:*
1292 + version: link:../packages/contracts
1293 + '@t3tools/provider-core':
1294 + specifier: workspace:*
1295 + version: link:../packages/provider-core
1296 + '@t3tools/shared':
1297 + specifier: workspace:*
1298 + version: link:../packages/shared
1299 + effect:
1300 + specifier: 4.0.5
1301 + version: 4.0.5
1302 + devDependencies:
1303 + '@effect/vitest':
1304 + specifier: 4.0.5
1305 + version: 4.0.5(patch_hash=2731c9c3761c595f25ddc625a7d61ca376b389d5b3049ecb837c8940f21cfa02)(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(effect@4.0.5)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0)
1306 + '@types/node':
1307 + specifier: 24.12.4
1308 + version: 24.12.4
1309 + typescript:
1310 + specifier: 'catalog:'
1311 + version: '@tsc-rs/typescript@0.8.0'
1312 + vite-plus:
1313 + specifier: 1.4.0
1314 + version: 1.4.0(@astrojs/compiler-rs@0.5.1)(@tsc-rs/typescript@0.8.0)(@types/node@24.12.4)(@vitest/browser-playwright@5.0.5)(@vitest/ui@5.0.5)(@volar/typescript@2.4.28(@tsc-rs/typescript@0.8.0))(bufferutil@4.1.0)(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(playwright@1.60.0)(terser@5.48.0)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0)
1315 +
1316 infra/relay:
1317 dependencies:
1318 '@clerk/backend':
pnpm-workspace.yaml
+1
@@ -1,5 +1,6 @@
1 packages:
2 - apps/*
3 + - evals
4 - infra/*
5 - oxlint-plugin-t3code
6 - packages/*