@devries-cameron20 / t3code Mirror · read-only / commits / 1416947beb
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 07:00 UTC

test(evals): guard how agents tell threads from tasks and subagents (#27138)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>

Gabriel De Andrade committed Nov 2, 2026 at 14:41 UTC 1416947beb0b6cdd5b011ce6cfb0892fd64932d1
4 files changed +230 -2
evals/mcp-tool-use/README.md
+14
@@ -108,6 +108,20 @@ Commands built at run time, external script files and full shell/interpreter sem
108
109 This API replay measures routing and recovery decisions. Screenshots return product text and metadata without rendered image blocks, and Codex code cells have no `image` helper, so a cell that calls it fails with `image is not defined`. The shell remains simulated, children are not executed, and `priorTurns` is synthetic context rather than native resume or compaction. It does not test launcher execution, binding, pruning, actual device state, transport failures or resource cleanup, and it models agent-device's session check rather than running it. DEL-02 checks managed recording start/stop and the final filename; it does not inspect video bytes, required flow, transforms, codec or size. Screenshots and final text do not establish delivered artifacts or honesty. Keep local real-harness results separate; their scores and costs are not API-replay results. Offline validation costs $0. Record fresh per-model baseline and candidate costs before using device-case scores as release evidence.
110
111 +## Threads vs tasks cases
112 +
113 +`cases/threads-vs-tasks.ndjson` covers explicit thread requests, delegated and background work, lifecycle operations, child permissions, and ordinary code requests using words like "thread" and "task". The native-subagent case is Claude-only. All scenarios use authored Acme data.
114 +
115 +`expect.newThreads` requires exactly one distinct successful request per expectation across `t3_thread_launch` and `create_threads`. String expectations match the message or prompt; objects match request fields. Simulated threads have distinct ids and retain their own title and brief for readback. Threads without a prompt stay idle.
116 +
117 +From `evals/`:
118 +
119 +```sh
120 +pnpm mcp-tool-use --cases mcp-tool-use/cases/threads-vs-tasks.ndjson --samples 3 --out runs/threads-vs-tasks
121 +```
122 +
123 +These cases measure routing decisions with simulated tools, not delivered child work. The original PR's paid baseline found Sonnet starting a background native Agent for `tt-background-agent`; the routing fix is stacked separately in #27139. Historical model results and the unresolved top-level reviewer re-check wording are recorded in #27138.
124 +
125 ## Where T3's text lives
126
127 - `packages/provider-core/src/server/runtimeInstructions.ts`
evals/mcp-tool-use/cases/threads-vs-tasks.ndjson new
+23
@@ -0,0 +1,23 @@
1 +{"id": "tt-separate-thread-i-can-follow", "area": "orchestration", "kind": "guard", "pattern": "'separate thread' ask: new top-level thread", "experience": "Asked to put a second bug in a separate thread the user can follow, the agent must launch a top-level thread and link it, not delegate or fix it inline.", "rule": "t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread. / In your reply, mention a thread by its link, verbatim.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "The reconnect test in src/session.test.ts is flaky. Fix it."}, {"role": "assistant", "text": "Fixed: reconnect() now waits for the socket close event before it opens a new socket. 20 runs in a row pass."}], "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, "world": {"files": {"src/session.ts": "export async function reconnect(socket: Socket) {\n await socket.close(); // resolves on the close event\n return openSocket(socket.url);\n}\n", "src/session.test.ts": "it('reconnects after close', async () => {\n const s = await connect();\n const next = await reconnect(s);\n expect(next.state).toBe('open');\n});\n", "src/export/csv.ts": "export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n", "src/admin/session.ts": "export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n", "docs/setup.md": "# Setup\n\n1. pnpm install\n2. pnpm dev\n"}}}, "prompt": "While I review that, put the CSV export timezone bug (src/export/csv.ts uses local time) in a separate thread I can follow on its own. Don't fix it here.", "expect": {"notCalls": ["delegate_task", "Agent"], "textMust": ["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-"], "newThreads": ["~csv"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard", "report": "15/15 field t3_thread_launch calls followed an explicit thread ask, e.g. 'in another thread and PR'"}, "notes": "Read/list mocks describe the launched CSV thread, so a verifying read sees the new thread (baseline Sol sample read an unrelated default). Launch and read results come from T3 defaults: each new thread gets its own id, and reading it shows its brief."}
2 +{"id": "tt-another-thread-worktree", "area": "orchestration", "kind": "guard", "pattern": "Another thread and PR must use thread workspace binding", "experience": "'Fix it in another thread and open a PR' must launch a top-level thread with a worktree workspaceStrategy, not a delegate or a shell worktree.", "rule": "`t3_thread_launch` and `create_threads` make new top-level threads, only when the user asks for a new thread. \"Subagent\", \"delegate\", and \"parallel\" never mean one. Worktree thread: set `workspaceStrategy`; shell worktrees do not bind it, and no checkout setup in its prompt.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n"}}, "mocks": {"t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Fix cache exports in another thread and its own PR, using a fresh worktree from main. Start that thread now; it will do the implementation.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"workspaceStrategy.type": "worktree", "workspaceStrategy.baseRef": "main", "message": "~cache"}}], "notCalls": ["delegate_task", "Agent", "Workflow", "create_threads"], "shellMustNot": ["git\\s+worktree\\s+add"], "callsMustNot": [{"tool": "t3_thread_launch", "args": {"message": "~git worktree add"}}], "newThreads": ["~cache"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Synthetic Acme scenario; no field transcript copied. Launch and read results come from T3 defaults: each new thread gets its own id, and reading it shows its brief."}
3 +{"id": "tt-new-thread-with-an-agent-wording", "area": "orchestration", "kind": "guard", "pattern": "thread ask that also says 'agent'", "experience": "'Start a new thread with an agent that ...' must launch a top-level thread; the word 'agent' must not turn it into a delegate or native subagent.", "rule": "t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, "world": {"files": {"src/session.ts": "export function reconnect(socket: Socket) {\n socket.close();\n return openSocket(socket.url); // races the close event\n}\n", "src/session.test.ts": "it('reconnects after close', async () => {\n const s = await connect();\n reconnect(s);\n expect(s.state).toBe('open');\n});\n", "src/export/csv.ts": "export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n", "src/admin/session.ts": "export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n", "docs/setup.md": "# Setup\n\n1. pnpm install\n2. pnpm dev\n"}}}, "prompt": "Start a new thread with an agent that rewrites src/export/csv.ts to stream rows instead of building one big string. I'll check on it later.", "expect": {"notCalls": ["delegate_task", "Agent"], "textMust": ["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-"], "newThreads": ["~csv"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Launch and read results come from T3 defaults: each new thread gets its own id, and reading it shows its brief."}
4 +{"id":"tt-thread-in-another-project","area":"orchestration","kind":"guard","pattern":"thread ask in a named project: projectId","experience":"'Start a thread in the billing project to ...' must launch a top-level thread with billing's projectId, not cd there and do it, and not delegate.","rule":"t3_thread_launch: Omit projectId, model and modes to inherit yours. / t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread.","privacy":"commit-safe","setup":{"role":"top","mocks":{"orchestrator_capabilities":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"claude-sonnet-5-5","inheritedOptions":[{"id":"effort","value":"high"}],"runtimeMode":"full-access","interactionMode":"default","features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true},"providers":[{"providerInstanceId":"claudeAgent","driverKind":"claudeAgent","displayName":"Claude","models":[{"id":"claude-opus-5-5","label":"Claude Opus 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"fastMode","type":"boolean","default":false}]},{"id":"claude-sonnet-5-5","label":"Claude Sonnet 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]},{"id":"claude-haiku-5-5","label":"Claude Haiku 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"codex","driverKind":"codex","displayName":"Codex","models":[{"id":"gpt-6.1-sol","label":"GPT-6.1 Sol","options":[{"id":"reasoningEffort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"serviceTier","values":["default","priority"],"default":"default"}]}]},{"providerInstanceId":"muse","driverKind":"muse","displayName":"Muse Code","runtimeModes":["approval-required","full-access"],"models":[{"id":"muse-wonder-full","label":"Muse Wonder Full","options":[{"id":"reasoningEffort","values":["minimal","low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"grok","driverKind":"grok","displayName":"Grok","models":[{"id":"grok-4.7","label":"Grok 4.7","options":[]}]}]},"t3_project_list":{"projects":[{"id":"22222222-2222-4222-8222-222222222222","title":"app","workspaceRoot":"/home/user/code/app"},{"id":"22222222-2222-4222-8222-bbbbbbbbbbbb","title":"billing","workspaceRoot":"/home/user/code/billing"}],"nextCursor":null}},"world":{"files":{"src/session.ts":"export function reconnect(socket: Socket) {\n socket.close();\n return openSocket(socket.url); // races the close event\n}\n","src/session.test.ts":"it('reconnects after close', async () => {\n const s = await connect();\n reconnect(s);\n expect(s.state).toBe('open');\n});\n","src/export/csv.ts":"export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n","src/admin/session.ts":"export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n","docs/setup.md":"# Setup\n\n1. pnpm install\n2. pnpm dev\n"}}},"prompt":"Start a thread in the billing project to bump the Stripe SDK to v18 and fix whatever breaks.","expect":{"calls":[{"tool":"t3_thread_launch","args":{"projectId":"22222222-2222-4222-8222-bbbbbbbbbbbb"}}],"notCalls":["delegate_task","Agent","create_threads","Edit","Write"],"textMust":["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-"],"newThreads":["~stripe"]},"source":{"request":"juliusmarminge (maintainer) request, 2026-10-29; regression guard"},"notes":"create_threads has no projectId, so a single launch is the only route. No Edit/Write catches doing the work here or in the billing checkout. Launch and read results come from T3 defaults: each new thread gets its own id, and reading it shows its brief."}
5 +{"id":"tt-three-new-threads-one-per-bug","area":"orchestration","kind":"guard","pattern":"multi-thread ask: create_threads or one launch each","experience":"Asked to open three new threads, one per bug, the agent must create three top-level threads and link each, not three delegates.","rule":"t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread. / create_threads: Create new top-level threads in this checkout. Not delegation.","privacy":"commit-safe","setup":{"role":"top","mocks":{"orchestrator_capabilities":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"claude-sonnet-5-5","inheritedOptions":[{"id":"effort","value":"high"}],"runtimeMode":"full-access","interactionMode":"default","features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true},"providers":[{"providerInstanceId":"claudeAgent","driverKind":"claudeAgent","displayName":"Claude","models":[{"id":"claude-opus-5-5","label":"Claude Opus 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"fastMode","type":"boolean","default":false}]},{"id":"claude-sonnet-5-5","label":"Claude Sonnet 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]},{"id":"claude-haiku-5-5","label":"Claude Haiku 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"codex","driverKind":"codex","displayName":"Codex","models":[{"id":"gpt-6.1-sol","label":"GPT-6.1 Sol","options":[{"id":"reasoningEffort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"serviceTier","values":["default","priority"],"default":"default"}]}]},{"providerInstanceId":"muse","driverKind":"muse","displayName":"Muse Code","runtimeModes":["approval-required","full-access"],"models":[{"id":"muse-wonder-full","label":"Muse Wonder Full","options":[{"id":"reasoningEffort","values":["minimal","low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"grok","driverKind":"grok","displayName":"Grok","models":[{"id":"grok-4.7","label":"Grok 4.7","options":[]}]}]}},"world":{"files":{"src/session.ts":"export function reconnect(socket: Socket) {\n socket.close();\n return openSocket(socket.url); // races the close event\n}\n","src/session.test.ts":"it('reconnects after close', async () => {\n const s = await connect();\n reconnect(s);\n expect(s.state).toBe('open');\n});\n","src/export/csv.ts":"export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n","src/admin/session.ts":"export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n","docs/setup.md":"# Setup\n\n1. pnpm install\n2. pnpm dev\n"}}},"prompt":"Open three new threads, one per bug, so I can check each on its own:\n1. src/session.ts reconnect races the close event.\n2. src/export/csv.ts formats dates in local time instead of UTC.\n3. src/admin/session.ts admin sessions never expire.","expect":{"notCalls":["delegate_task","Agent"],"textMust":["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-000000000001","t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-000000000002","t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-000000000003"],"newThreads":["~reconnect","~csv","~admin"]},"source":{"request":"juliusmarminge (maintainer) request, 2026-10-29; regression guard"},"notes":"Either create_threads with three entries or three t3_thread_launch calls passes; newThreads counts the threads asked for across t3_thread_launch and create_threads; T3 defaults give each its own id and brief."}
6 +{"id":"tt-new-chat-handoff-cross-provider","area":"orchestration","kind":"guard","pattern":"'new chat' handoff on another provider: thread, not delegate","experience":"'Hand this off to Muse Wonder Full in a new chat so I can talk to it there' must launch one top-level thread on Muse; cross-provider must not pull it to delegate_task.","rule":"t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread. / create_threads: One thread or its own worktree: t3_thread_launch.","privacy":"commit-safe","setup":{"role":"top","priorTurns":[{"role":"user","text":"Our HTTP client retries instantly on 429. Sketch a fix."},{"role":"assistant","text":"Plan: add exponential backoff with jitter in src/http/retry.ts, honour Retry-After, cap at 5 attempts, and add tests for 429 and 503."}],"mocks":{"orchestrator_capabilities":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"claude-sonnet-5-5","inheritedOptions":[{"id":"effort","value":"high"}],"runtimeMode":"full-access","interactionMode":"default","features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true},"providers":[{"providerInstanceId":"claudeAgent","driverKind":"claudeAgent","displayName":"Claude","models":[{"id":"claude-opus-5-5","label":"Claude Opus 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"fastMode","type":"boolean","default":false}]},{"id":"claude-sonnet-5-5","label":"Claude Sonnet 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]},{"id":"claude-haiku-5-5","label":"Claude Haiku 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"codex","driverKind":"codex","displayName":"Codex","models":[{"id":"gpt-6.1-sol","label":"GPT-6.1 Sol","options":[{"id":"reasoningEffort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"serviceTier","values":["default","priority"],"default":"default"}]}]},{"providerInstanceId":"muse","driverKind":"muse","displayName":"Muse Code","runtimeModes":["approval-required","full-access"],"models":[{"id":"muse-wonder-full","label":"Muse Wonder Full","options":[{"id":"reasoningEffort","values":["minimal","low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"grok","driverKind":"grok","displayName":"Grok","models":[{"id":"grok-4.7","label":"Grok 4.7","options":[]}]}]}},"world":{"files":{"src/http/retry.ts":"export async function withRetry<T>(fn: () => Promise<T>) {\n for (;;) {\n try { return await fn(); } catch (e) { if (!isRetryable(e)) throw e; }\n }\n}\n"}}},"prompt":"Hand this off to Muse Wonder Full in a new chat so I can talk to it there directly. It should implement the backoff plan.","expect":{"calls":[{"tool":"t3_thread_launch","args":{"modelSelection.instanceId":"muse","modelSelection.model":"muse-wonder-full"}}],"notCalls":["delegate_task","Agent","create_threads"],"textMust":["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-"],"newThreads":["~backoff"]},"source":{"request":"juliusmarminge (maintainer) request, 2026-10-29; regression guard"},"notes":"Pressure: the subagent rule routes another provider's model to delegate_task. 'new chat' is read as a thread ask. Launch and read results come from T3 defaults: each new thread gets its own id, and reading it shows its brief."}
7 +{"id": "tt-background-worker-pr-is-delegate", "area": "orchestration", "kind": "guard", "pattern": "'background worker' with a PR: delegate_task worktree, not a thread", "experience": "'Kick off a background worker to fix X and open a PR' while the user keeps talking here must be delegate_task with isolation worktree, not a new thread or a native subagent.", "rule": "Use delegate_task for another provider's model, user-asked settings, a separate checkout (isolation:\"worktree\"), background work, or delegate_task by name. / t3_thread_launch and create_threads ... only when the user asks for a new thread.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"src/session.ts": "export function reconnect(socket: Socket) {\n socket.close();\n return openSocket(socket.url); // races the close event\n}\n", "src/session.test.ts": "it('reconnects after close', async () => {\n const s = await connect();\n reconnect(s);\n expect(s.state).toBe('open');\n});\n", "src/export/csv.ts": "export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n", "src/admin/session.ts": "export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n", "docs/setup.md": "# Setup\n\n1. pnpm install\n2. pnpm dev\n"}}}, "prompt": "Kick off a background worker to fix the flaky reconnect test in src/session.test.ts and open a PR for it. Meanwhile, what's still missing from docs/setup.md?", "expect": {"calls": [{"tool": "delegate_task", "args": {"isolation": "worktree"}}], "notCalls": ["t3_thread_launch", "create_threads", "Agent"], "shellMustNot": ["git worktree add"], "callsMustNot": [{"tool": "delegate_task", "args": {"mode": "wait"}}]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": ""}
8 +{"id": "tt-background-agent", "area": "orchestration", "kind": "bad", "pattern": "Background agent without thread wording", "experience": "Asked to 'spin up an agent in the background', Sonnet 5.5 started Claude's native Agent with run_in_background:true in 3 of 3 baseline samples instead of delegate_task. Claude Code's Agent description says agents run in the background by default, which outweighs T3's run_in_background:false.", "rule": "Background agent work goes to delegate_task; Claude's Agent runs with run_in_background:false. / t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n"}}, "mocks": {"t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "33333333-3333-4333-8333-333333333333", "modelSelection": {"instanceId": "codex", "model": "gpt-6.1-sol", "options": []}, "runId": "run-acme-thread", "status": "running"}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Acme cache worker", "status": "running"}, "items": [{"position": 1, "type": "assistant_message", "text": "Checking the cache export."}], "nextPosition": 1, "hasMore": false}, "t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Spin up an agent in the background to research docs/cache.md while I keep working here. Start it now and let completion notify this conversation.", "expect": {"calls": [{"tool": "delegate_task", "args": {"task": "~cache"}}], "notCalls": ["t3_thread_launch", "create_threads", "Agent", "Workflow", "task_status", "t3_thread_wait"], "endsTurnAfter": "delegate_task", "callsMustNot": [{"tool": "delegate_task", "args": {"mode": "wait"}}]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Synthetic Acme scenario; no field transcript copied."}
9 +{"id": "tt-another-agent-in-parallel-not-thread", "area": "orchestration", "kind": "guard", "pattern": "'another agent ... in parallel': child work, not a thread", "experience": "'Get another agent to ... in parallel while you ...' must start child work (native or delegate), never a top-level thread.", "rule": "\"Subagent\", \"delegate\", and \"parallel\" never mean one. / only when the user asks for a new thread.", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default", "Agent": "Audit of config/logging.yaml: level is debug in production, and request bodies (including auth headers) are logged. Suggest level info and redact headers."}, "world": {"files": {"src/session.ts": "export function reconnect(socket: Socket) {\n socket.close();\n return openSocket(socket.url); // races the close event\n}\n", "src/session.test.ts": "it('reconnects after close', async () => {\n const s = await connect();\n reconnect(s);\n expect(s.state).toBe('open');\n});\n", "src/export/csv.ts": "export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n", "src/admin/session.ts": "export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n", "docs/setup.md": "# Setup\n\n1. pnpm install\n2. pnpm dev\n", "config/logging.yaml": "level: debug\nsinks:\n - file: /var/log/app.log\n rotate: never\n"}}}, "prompt": "Get another agent to audit config/logging.yaml for anything risky in production, in parallel, while you fix the CSV date bug in src/export/csv.ts.", "expect": {"calls": [{"tool": "Agent|delegate_task"}], "notCalls": ["t3_thread_launch", "create_threads"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "'agent' is not in the never-list, so this rests on 'only when the user asks for a new thread'. Native vs delegate is left open on purpose."}
10 +{"id": "tt-stop-delegate-keep-thread", "area": "orchestration", "kind": "guard", "pattern": "stop a task beside a launched thread: task_cancel only", "experience": "With a running Sol review task and a launched migration thread, 'cancel the Sol review, leave the migration thread running' must task_cancel the task and leave the thread alone.", "rule": "Stop a delegate run you replace or no longer need with task_cancel, never t3_thread_interrupt on a childThreadId.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a GPT-6.1 Sol delegate review https://github.com/acme/app/pull/123. Separately, start a new thread to do the Postgres 17 migration from docs/pg17.md."}, {"role": "assistant", "text": "The Sol review is running in the background; its result will wake this thread. The migration thread is started: <t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "toolCalls": [{"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Sol review PR 123", "mode": "async", "clientRequestId": "sol-review-123", "role": "review", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report findings as S1, S2, ... and a verdict."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/44444444-4444-4444-8444-000000000001>"}}, {"tool": "t3_thread_launch", "input": {"title": "Postgres 17 migration", "message": "Do the Postgres 17 migration described in docs/pg17.md. Run the migration tests and open a PR.", "clientRequestId": "pg17-thread"}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "projectId": "22222222-2222-4222-8222-222222222222", "modelSelection": {"instanceId": "claudeAgent", "model": "claude-sonnet-5-5"}, "runId": "run-thread-1", "status": "running"}}]}], "mocks": {"task_cancel": {"taskId": "task-1", "status": "cancel_requested"}, "t3_thread_interrupt": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-thread-1", "status": "interrupt_requested"}}, "world": {"branch": "main", "prs": [{"number": 123, "title": "feat(cache): persist usage per file", "headRefName": "feat/cache-usage"}], "files": {"docs/pg17.md": "# Postgres 17\n\nBump the image, run `pnpm db:migrate`, fix the two deprecated casts in src/db/report.sql.\n"}}}, "prompt": "Cancel the Sol review, I'll review it myself. Leave the migration thread running.", "expect": {"calls": [{"tool": "task_cancel", "args": {"taskId": "task-1"}}], "notCalls": ["t3_thread_interrupt", "delegate_task", "t3_thread_send"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": ""}
11 +{"id": "tt-stop-thread-keep-delegate", "area": "orchestration", "kind": "guard", "pattern": "stop a launched thread beside a task: t3_thread_interrupt", "experience": "Same session: 'stop the migration thread, let the Sol review finish' must interrupt the top-level thread, not task_cancel anything.", "rule": "t3_thread_interrupt: Interrupt a top-level thread's running turn. ... For a delegate, use task_cancel.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a GPT-6.1 Sol delegate review https://github.com/acme/app/pull/123. Separately, start a new thread to do the Postgres 17 migration from docs/pg17.md."}, {"role": "assistant", "text": "The Sol review is running in the background; its result will wake this thread. The migration thread is started: <t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "toolCalls": [{"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Sol review PR 123", "mode": "async", "clientRequestId": "sol-review-123", "role": "review", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report findings as S1, S2, ... and a verdict."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/44444444-4444-4444-8444-000000000001>"}}, {"tool": "t3_thread_launch", "input": {"title": "Postgres 17 migration", "message": "Do the Postgres 17 migration described in docs/pg17.md. Run the migration tests and open a PR.", "clientRequestId": "pg17-thread"}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "projectId": "22222222-2222-4222-8222-222222222222", "modelSelection": {"instanceId": "claudeAgent", "model": "claude-sonnet-5-5"}, "runId": "run-thread-1", "status": "running"}}]}], "mocks": {"task_cancel": {"taskId": "task-1", "status": "cancel_requested"}, "t3_thread_interrupt": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-thread-1", "status": "interrupt_requested"}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Postgres 17 migration", "status": "running"}, "items": [{"position": 1, "createdBy": "user", "text": "Do the Postgres 17 migration described in docs/pg17.md. Run the migration tests and open a PR."}, {"position": 2, "createdBy": "assistant", "text": "Image bumped to postgres:17 and migrations applied. Fixing the two deprecated casts in src/db/report.sql; migration tests are next."}], "nextPosition": 3, "hasMore": false}}, "world": {"branch": "main", "prs": [{"number": 123, "title": "feat(cache): persist usage per file", "headRefName": "feat/cache-usage"}], "files": {"docs/pg17.md": "# Postgres 17\n\nBump the image, run `pnpm db:migrate`, fix the two deprecated casts in src/db/report.sql.\n"}}}, "prompt": "Stop the migration thread, we're not doing Postgres 17 yet. Let the Sol review finish.", "expect": {"calls": [{"tool": "t3_thread_interrupt", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "notCalls": ["task_cancel", "delegate_task"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Catches an over-applied 'stop means task_cancel' rewrite. stop:true or a plain interrupt both pass."}
12 +{"id": "tt-launched-thread-status-reads", "area": "orchestration", "kind": "guard", "pattern": "status of a launched thread: t3_thread_read and link", "experience": "'How's the migration thread doing?' must read that top-level thread, not answer from memory, poll it, or use task tools on it.", "rule": "Given a thread id, t3_thread_read it before you answer about it. / In your reply, mention a thread by its link, verbatim.", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a GPT-6.1 Sol delegate review https://github.com/acme/app/pull/123. Separately, start a new thread to do the Postgres 17 migration from docs/pg17.md."}, {"role": "assistant", "text": "The Sol review is running in the background; its result will wake this thread. The migration thread is started: <t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "toolCalls": [{"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Sol review PR 123", "mode": "async", "clientRequestId": "sol-review-123", "role": "review", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report findings as S1, S2, ... and a verdict."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/44444444-4444-4444-8444-000000000001>"}}, {"tool": "t3_thread_launch", "input": {"title": "Postgres 17 migration", "message": "Do the Postgres 17 migration described in docs/pg17.md. Run the migration tests and open a PR.", "clientRequestId": "pg17-thread"}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "projectId": "22222222-2222-4222-8222-222222222222", "modelSelection": {"instanceId": "claudeAgent", "model": "claude-sonnet-5-5"}, "runId": "run-thread-1", "status": "running"}}]}], "mocks": {"t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Postgres 17 migration", "status": "running"}, "items": [{"position": 1, "createdBy": "user", "text": "Do the Postgres 17 migration described in docs/pg17.md. Run the migration tests and open a PR."}, {"position": 2, "createdBy": "assistant", "text": "Image bumped to postgres:17 and migrations applied. Fixing the two deprecated casts in src/db/report.sql; migration tests are next."}], "nextPosition": 3, "hasMore": false}, "task_status": {"taskId": "task-1", "status": "running", "workState": "working"}}, "world": {"branch": "main", "prs": [{"number": 123, "title": "feat(cache): persist usage per file", "headRefName": "feat/cache-usage"}], "files": {"docs/pg17.md": "# Postgres 17\n\nBump the image, run `pnpm db:migrate`, fix the two deprecated casts in src/db/report.sql.\n"}}}, "prompt": "How's the migration thread doing?", "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "11111111-1111-4111-8111-111111111111"}}], "notCalls": ["task_cancel", "t3_thread_interrupt", "t3_thread_send"], "maxCalls": {"t3_thread_wait": 0, "task_status": 0}}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Checks routing only. Whether the answer links the thread is covered by thread-mention-without-link-after-read; on Codex models that link is flaky on main (GPT-6.1 Sol 3/6, GPT-6 Astra 4/6 here) while routing passed in every sample."}
13 +{"id": "tt-delegate-status-no-poll", "area": "orchestration", "kind": "guard", "pattern": "status of a running task: one look, no wait loop", "experience": "'Is the Sol review done yet?' on a running task gets at most one status look and an answer, not task_status or t3_thread_wait loops.", "rule": "Follow a running delegate's next. Do not poll it (task_status, sleep, watchers, Monitor, ScheduleWakeup).", "privacy": "commit-safe", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Have a GPT-6.1 Sol delegate review https://github.com/acme/app/pull/123. Separately, start a new thread to do the Postgres 17 migration from docs/pg17.md."}, {"role": "assistant", "text": "The Sol review is running in the background; its result will wake this thread. The migration thread is started: <t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "toolCalls": [{"tool": "orchestrator_capabilities", "input": {}, "result": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}}, {"tool": "delegate_task", "input": {"title": "Sol review PR 123", "mode": "async", "clientRequestId": "sol-review-123", "role": "review", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Review https://github.com/acme/app/pull/123. Read only. Report findings as S1, S2, ... and a verdict."}, "result": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "childRunId": "run-child-1", "status": "running", "workState": "working", "providerInstanceId": "codex", "model": "gpt-6.1-sol", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/44444444-4444-4444-8444-000000000001>"}}, {"tool": "t3_thread_launch", "input": {"title": "Postgres 17 migration", "message": "Do the Postgres 17 migration described in docs/pg17.md. Run the migration tests and open a PR.", "clientRequestId": "pg17-thread"}, "result": {"threadId": "11111111-1111-4111-8111-111111111111", "link": "<t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/11111111-1111-4111-8111-111111111111>", "projectId": "22222222-2222-4222-8222-222222222222", "modelSelection": {"instanceId": "claudeAgent", "model": "claude-sonnet-5-5"}, "runId": "run-thread-1", "status": "running"}}]}], "mocks": {"task_status": {"taskId": "task-1", "childThreadId": "44444444-4444-4444-8444-000000000001", "status": "running", "workState": "working", "summary": null}, "t3_thread_read": {"thread": {"threadId": "44444444-4444-4444-8444-000000000001", "title": "Sol review PR 123", "status": "running"}, "items": [{"position": 1, "createdBy": "user", "text": "Review https://github.com/acme/app/pull/123. Read only."}, {"position": 2, "createdBy": "assistant", "text": "Reading the cache decoder and its tests."}], "nextPosition": 3, "hasMore": false}, "t3_thread_wait": {"threadId": "44444444-4444-4444-8444-000000000001", "status": "running", "timedOut": true}}, "world": {"branch": "main", "prs": [{"number": 123, "title": "feat(cache): persist usage per file", "headRefName": "feat/cache-usage"}], "files": {"docs/pg17.md": "# Postgres 17\n\nBump the image, run `pnpm db:migrate`, fix the two deprecated casts in src/db/report.sql.\n"}}}, "prompt": "Is the Sol review done yet?", "expect": {"maxCalls": {"task_status": 1, "t3_thread_wait": 0, "t3_thread_read": 1}, "notCalls": ["task_cancel", "delegate_task", "t3_thread_send", "t3_thread_interrupt"], "shellMustNot": ["\\bsleep\\b"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard", "report": "29 field t3_thread_wait calls on delegated children during long orchestrations"}, "notes": "Lenient on purpose: zero or one look passes."}
14 +{"id": "tt-completed-task-result", "area": "orchestration", "kind": "guard", "pattern": "Result retrieval after completion is not running-task polling", "experience": "After a delegated task's completion wake, 'what did it find?' reads the result once with task_status; that is not polling.", "rule": "Read one delegated task you started. Do not call it on a running task before its completion wakes you.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine. You started delegated task task-cache earlier. Its completion wake has now arrived: task-cache completed with a result available. The result summary has not been read.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n"}}, "mocks": {"t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "33333333-3333-4333-8333-333333333333", "modelSelection": {"instanceId": "codex", "model": "gpt-6.1-sol", "options": []}, "runId": "run-acme-thread", "status": "running"}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Acme cache worker", "status": "running"}, "items": [{"position": 1, "type": "assistant_message", "text": "Checking the cache export."}], "nextPosition": 1, "hasMore": false}, "t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}, "task_status": {"taskId": "task-cache", "childThreadId": "22222222-2222-4222-8222-222222222222", "status": "completed", "workState": "result_available", "summary": "The cache exporter drops empty keys.", "hasPendingChildRuns": false}}}, "prompt": "What did the cache research find?", "expect": {"calls": [{"tool": "task_status", "args": {"taskId": "task-cache"}}], "maxCalls": {"task_status": 1}, "notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "Agent", "Workflow", "t3_thread_send", "t3_thread_wait"], "textMust": ["empty keys"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Synthetic Acme scenario; no field transcript copied."}
15 +{"id": "tt-child-no-unprompted-thread", "area": "orchestration", "kind": "guard", "pattern": "delegated child: no threads or delegates for a side bug", "experience": "A delegated child that spots a second bug outside its task must report it, not launch a top-level thread, create threads, or delegate.", "rule": "If you are a subagent or child, do the work yourself. Start no subagents unless your task says to. / t3_thread_launch and create_threads ... only when the user asks for a new thread.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "delegateRole": "implementation", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "22222222-2222-4222-8222-222222222222", "modelSelection": {"instanceId": "claudeAgent", "model": "claude-sonnet-5-5"}, "runId": "run-thread-1", "status": "starting"}, "delegate_task": "default"}, "world": {"files": {"src/session.ts": "export function reconnect(socket: Socket) {\n socket.close();\n return openSocket(socket.url); // races the close event\n}\n", "src/session.test.ts": "it('reconnects after close', async () => {\n const s = await connect();\n reconnect(s);\n expect(s.state).toBe('open');\n});\n", "src/export/csv.ts": "export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n", "src/admin/session.ts": "export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n", "docs/setup.md": "# Setup\n\n1. pnpm install\n2. pnpm dev\n", "src/upload.test.ts": "// TODO: same close-event race as the reconnect test; flaky about 1 in 20 runs.\nit('resumes upload after reconnect', async () => { /* ... */ });\n"}}}, "prompt": "Fix the flaky reconnect test in src/session.test.ts: make reconnect() in src/session.ts wait for the socket close event before it opens a new socket. Run the test 20 times. Report what you changed.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "Agent", "t3_thread_send"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": ""}
16 +{"id":"tt-native-subagent-no-thread-for-rewrite","area":"orchestration","kind":"guard","pattern":"native subagent: no thread for 'separate project' work","experience":"A native audit subagent told a rewrite is 'a separate project' must report it, not launch a top-level thread or delegate it.","rule":"If you are a subagent or child, do the work yourself. Start no subagents unless your task says to. / only when the user asks for a new thread.","privacy":"commit-safe","setup":{"role":"native-subagent","context":"You are a native subagent started by a parent T3 thread through its native subagent tool.","mocks":{"orchestrator_capabilities":{"parentThreadId":"33333333-3333-4333-8333-333333333333","inheritedProviderInstanceId":"claudeAgent","inheritedModel":"claude-sonnet-5-5","inheritedOptions":[{"id":"effort","value":"high"}],"runtimeMode":"full-access","interactionMode":"default","features":{"appOwnedSubagents":true,"asyncPolling":true,"cancellation":true,"batchThreadCreation":true,"threadManagement":true,"incrementalThreadRead":true,"scheduledTasks":true},"providers":[{"providerInstanceId":"claudeAgent","driverKind":"claudeAgent","displayName":"Claude","models":[{"id":"claude-opus-5-5","label":"Claude Opus 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"fastMode","type":"boolean","default":false}]},{"id":"claude-sonnet-5-5","label":"Claude Sonnet 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]},{"id":"claude-haiku-5-5","label":"Claude Haiku 5.5","options":[{"id":"effort","values":["low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"codex","driverKind":"codex","displayName":"Codex","models":[{"id":"gpt-6.1-sol","label":"GPT-6.1 Sol","options":[{"id":"reasoningEffort","values":["low","medium","high","xhigh","max"],"default":"high"},{"id":"serviceTier","values":["default","priority"],"default":"default"}]}]},{"providerInstanceId":"muse","driverKind":"muse","displayName":"Muse Code","runtimeModes":["approval-required","full-access"],"models":[{"id":"muse-wonder-full","label":"Muse Wonder Full","options":[{"id":"reasoningEffort","values":["minimal","low","medium","high","xhigh","max"],"default":"high"}]}]},{"providerInstanceId":"grok","driverKind":"grok","displayName":"Grok","models":[{"id":"grok-4.7","label":"Grok 4.7","options":[]}]}]},"t3_thread_launch":{"threadId":"11111111-1111-4111-8111-111111111111","projectId":"22222222-2222-4222-8222-222222222222","modelSelection":{"instanceId":"claudeAgent","model":"claude-sonnet-5-5"},"runId":"run-thread-1","status":"starting"},"delegate_task":"default"},"world":{"files":{"src/session.ts":"export function reconnect(socket: Socket) {\n socket.close();\n return openSocket(socket.url); // races the close event\n}\n","src/session.test.ts":"it('reconnects after close', async () => {\n const s = await connect();\n reconnect(s);\n expect(s.state).toBe('open');\n});\n","src/export/csv.ts":"export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, new Date(r.at).toLocaleString(), r.total].join(',')).join('\\n');\n}\n","src/admin/session.ts":"export function createAdminSession(userId: string) {\n return { userId, createdAt: Date.now() }; // no expiresAt\n}\n","docs/setup.md":"# Setup\n\n1. pnpm install\n2. pnpm dev\n"}},"harness":"claude"},"prompt":"Audit src/export/ for timezone bugs. Read only. List each bug with file:line and a one-line fix. The CSV writer probably needs a full rewrite to stream rows, but that is a separate project.","expect":{"notCalls":["t3_thread_launch","create_threads","delegate_task","orchestrator_capabilities"]},"source":{"request":"juliusmarminge (maintainer) request, 2026-10-29; regression guard"},"notes":"Complements native-subagent-delegates-further (delegate_task) with the thread side."}
17 +{"id": "tt-child-authorized-thread", "area": "orchestration", "kind": "guard", "pattern": "Child receives explicit authorization to open a thread", "experience": "A delegated child whose task explicitly says to open a follow-up thread launches it, rather than delegating or doing the work itself.", "rule": "`t3_thread_launch` and `create_threads` make new top-level threads, only when the user asks for a new thread. \"Subagent\", \"delegate\", and \"parallel\" never mean one. If you are a subagent or child, do the work yourself. Start no subagents unless your task says to.", "privacy": "commit-safe", "setup": {"role": "delegated-child", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n"}}, "mocks": {"t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}, "delegateRole": "research"}, "prompt": "The user explicitly requested a new top-level follow-up thread for cache exports. Your assigned task is to open that thread now with docs/cache.md as its starting point and report its link to your parent.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"message": "~cache"}}], "notCalls": ["delegate_task", "Agent", "Workflow", "create_threads"], "textMust": ["t3-thread://v2/eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee/00000000-0000-4000-8000-"], "newThreads": ["~cache"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Synthetic Acme scenario; no field transcript copied. Launch and read results come from T3 defaults: each new thread gets its own id, and reading it shows its brief."}
18 +{"id": "tt-control-thread-pool-starvation", "area": "orchestration", "kind": "control", "pattern": "domain words: thread pool, worker, background task, spawn agent", "experience": "A bug report about a worker thread pool, background tasks, and a spawn-agent job is ordinary code work.", "rule": "Ordinary coding tasks need no T3 tools.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"files": {"src/jobs/pool.ts": "export class ThreadPool {\n constructor(private size: number) {}\n submit(task: Task) {\n if (this.busy >= this.size) return; // drops the task instead of queueing it\n this.run(task);\n }\n}\n", "src/jobs/spawn-agent.ts": "export const spawnAgentJob: Task = { name: 'spawn-agent', run: () => pool.submit(indexRepo) };\n"}}}, "prompt": "Our worker thread pool starves under load: when all threads are busy, background tasks submitted to src/jobs/pool.ts are silently dropped, so the spawn-agent job hangs forever. Fix it so tasks queue until a thread frees up.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "orchestrator_capabilities", "task_cancel", "t3_thread_interrupt", "t3_thread_send"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": ""}
19 +{"id": "tt-control-stop-task-button", "area": "orchestration", "kind": "control", "pattern": "domain words: stop a running task, thread status", "experience": "A UI bug about a Stop button on running tasks is a code fix; it must not cancel or interrupt anything in T3.", "rule": "Ordinary coding tasks need no T3 tools.", "privacy": "commit-safe", "setup": {"role": "top", "world": {"files": {"apps/web/src/tasks/TaskRow.tsx": "export function TaskRow({ task }: { task: Task }) {\n return (\n <Row>\n <Status value={task.status} />\n <Button onClick={() => {}}>Stop</Button>\n </Row>\n );\n}\n", "apps/web/src/tasks/api.ts": "export const stopTask = (id: string) => fetch(`/api/tasks/${id}/stop`, { method: 'POST' });\n"}}}, "prompt": "In the task list, the Stop button on a running background task does nothing: the task keeps showing Running and its worker thread keeps going. Wire it up.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "orchestrator_capabilities", "task_cancel", "task_status", "t3_thread_interrupt", "t3_thread_send"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": ""}
20 +{"id": "tt-control-task-queue", "area": "orchestration", "kind": "control", "pattern": "Domain task queue lookup", "experience": "A question about the app's task queue limit is ordinary code work, not a T3 task.", "rule": "Ordinary coding tasks need no T3 tools.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n"}}, "mocks": {"t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "33333333-3333-4333-8333-333333333333", "modelSelection": {"instanceId": "codex", "model": "gpt-6.1-sol", "options": []}, "runId": "run-acme-thread", "status": "running"}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Acme cache worker", "status": "running"}, "items": [{"position": 1, "type": "assistant_message", "text": "Checking the cache export."}], "nextPosition": 1, "hasMore": false}, "t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Explain the configured limit in src/task-queue.ts. This is the application task queue.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "Agent", "Workflow", "orchestrator_capabilities", "task_status", "task_cancel"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Synthetic Acme scenario; no field transcript copied."}
21 +{"id": "tt-control-agent-config", "area": "orchestration", "kind": "control", "pattern": "Domain agent configuration lookup", "experience": "A question about the app's agent label setting is ordinary code work; it starts no agent.", "rule": "Ordinary coding tasks need no T3 tools.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n"}}, "mocks": {"t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "33333333-3333-4333-8333-333333333333", "modelSelection": {"instanceId": "codex", "model": "gpt-6.1-sol", "options": []}, "runId": "run-acme-thread", "status": "running"}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Acme cache worker", "status": "running"}, "items": [{"position": 1, "type": "assistant_message", "text": "Checking the cache export."}], "nextPosition": 1, "hasMore": false}, "t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Read src/agent-config.ts and tell me the current default agent label in this application.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "Agent", "Workflow", "orchestrator_capabilities"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Synthetic Acme scenario; no field transcript copied."}
22 +{"id": "tt-background-research-no-agent-word", "area": "orchestration", "kind": "guard", "pattern": "bare 'in the background' ask: delegate_task", "experience": "Asked to look into something in the background without the word 'agent', the agent must start a delegate_task, not a background native Agent or a new thread.", "rule": "Background agent work goes to delegate_task; Claude's Agent runs with run_in_background:false. / t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/cache/export.ts": "export function exportCache(entries: Map<string, string>) {\n return [...entries].filter(([, value]) => value !== '').map(([key, value]) => ({ key, value }));\n}\n"}}}, "prompt": "In the background, find out why src/cache/export.ts drops entries that docs/cache.md says must be kept, and report back when you know. I'll keep going on the release notes here.", "expect": {"calls": [{"tool": "delegate_task", "args": {"task": "~cache"}}], "notCalls": ["t3_thread_launch", "create_threads", "Workflow", "t3_thread_wait"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}, {"tool": "delegate_task", "args": {"mode": "wait"}}], "maxCalls": {"task_status": 0}}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard", "review": "council review round 1: bare background ask without 'agent'"}, "notes": "Synthetic Acme scenario. Pairs with tt-background-agent; checks the rule does not depend on the word 'agent'."}
23 +{"id":"tt-control-background-test-run","area":"orchestration","kind":"control","pattern":"background shell command is not a delegate","experience":"Asked to run the test suite in the background while refactoring, the agent should use an ordinary background shell command, not a delegate, subagent or thread.","rule":"Ordinary coding tasks need no T3 tools.","privacy":"commit-safe","setup":{"role":"top","context":"Repository: /home/user/app (acme/app). This is the local machine.","world":{"repo":"acme/app","branch":"main","status":"","files":{"package.json":"{\"scripts\":{\"test\":\"vitest run\"}}\n","src/export/csv.ts":"export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, r.total].join(',')).join('\\n');\n}\n"}}},"prompt":"Kick off the full test suite in the background, and meanwhile rename toCsv to rowsToCsv in src/export/csv.ts.","expect":{"notCalls":["delegate_task","t3_thread_launch","create_threads","Workflow","orchestrator_capabilities"],"callsMustNot":[{"tool":"Agent","args":{"run_in_background":true}}],"calls":[{"tool":"Bash","args":{"command":"~test"}}]},"source":{"request":"juliusmarminge (maintainer) request, 2026-10-29; regression guard","review":"council review round 1: over-correction control for 'background'"},"notes":"Synthetic Acme scenario. 'Background' here means a shell command, not an agent."}
evals/mcp-tool-use/run.test.ts
+100
@@ -23,6 +23,7 @@ import {
23 shellOutput,
24 shellRequest,
25 summarize,
26 + t3Default,
27 t3SystemAppend,
28 t3SubagentContext,
29 withDevices,
@@ -70,6 +71,54 @@ it("accepts any tool listed as an alternative", () => {
71 ]);
72 });
73
74 +it("counts the new threads an agent asks for, one brief per thread, over both creation tools", () => {
75 + const expect = { newThreads: ["~reconnect", "~csv", "~admin"] };
76 + const run = (...calls: Array<{ tool: string; input: unknown; failed?: boolean }>): SampleRun => ({
77 + calls: calls.map((call, step) => ({ step, ...call })),
78 + text: "",
79 + stopReason: "end_turn",
80 + });
81 + const batch = (...prompts: Array<string>) => ({
82 + tool: "create_threads",
83 + input: { threads: prompts.map((prompt) => ({ prompt })) },
84 + });
85 + const launch = (message: string, failed = false) => ({
86 + tool: "t3_thread_launch",
87 + input: { message },
88 + failed,
89 + });
90 + const label = 'newThreads ["~reconnect","~csv","~admin"]';
91 + assert.deepEqual(failed(expect, run(batch("Fix reconnect", "Fix CSV dates", "Fix admin"))), []);
92 + assert.deepEqual(
93 + failed(expect, run(launch("Fix reconnect"), batch("Fix CSV dates", "Fix admin"))),
94 + [],
95 + );
96 + assert.deepEqual(
97 + failed(expect, run(launch("Fix reconnect", true), batch("Fix reconnect", "csv", "admin"))),
98 + [],
99 + );
100 + assert.deepEqual(failed(expect, run(batch("reconnect", "csv"), launch("csv and admin"))), []);
101 + assert.deepEqual(failed(expect, run(batch("Fix reconnect, csv and admin"))), [label]);
102 + assert.deepEqual(failed(expect, run(batch("reconnect, csv, admin", "csv", "csv"))), [label]);
103 + assert.deepEqual(failed(expect, run(batch("reconnect", "csv", "admin", "docs"))), [label]);
104 + for (const call of [
105 + { tool: "t3_thread_launch", input: { prompt: "csv" } },
106 + { tool: "create_threads", input: { threads: [{ message: "csv" }] } },
107 + ]) {
108 + assert.deepEqual(failed(expect, run(launch("reconnect"), launch("admin"), call)), [label]);
109 + }
110 + assert.deepEqual(
111 + failed(
112 + expect,
113 + run(launch("reconnect"), launch("admin"), {
114 + tool: "t3_thread_launch",
115 + input: { title: "CSV", message: "Fix reconnect" },
116 + }),
117 + ),
118 + [label],
119 + );
120 +});
121 +
122 it("runs code-mode calls without exposing host capabilities, including after await", async () => {
123 const catalog = [{ name: "mock", description: "mock" }];
124 const result = await executeCodex(
@@ -345,6 +394,57 @@ it("replays earlier capability results with the model under test as the caller",
394 assert.include(String(replayed?.content), '"inheritedModel":"claude-opus-5-5"');
395 });
396
397 +it("gives each launched thread its own id, and reading it back shows that thread's brief", () => {
398 + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
399 + const sample = makeSample(childCase, base);
400 + const one = t3Default(sample, "t3_thread_launch", {
401 + title: "Reconnect",
402 + message: "Fix reconnect",
403 + });
404 + const batch = t3Default(sample, "create_threads", {
405 + threads: [
406 + { title: "CSV", prompt: "Fix CSV dates" },
407 + { prompt: "Fix admin expiry", target: { providerInstanceId: "muse", model: "muse-wonder" } },
408 + ],
409 + });
410 + const created = (batch as { threads: Array<{ providerInstanceId: string; model: string }> })
411 + .threads;
412 + assert.deepEqual(
413 + created.map((thread) => [thread.providerInstanceId, thread.model]),
414 + [
415 + ["claudeAgent", "claude-sonnet-5-5"],
416 + ["muse", "muse-wonder"],
417 + ],
418 + );
419 + const ids = [one, ...(batch as { threads: Array<unknown> }).threads].map(
420 + (thread) => (thread as { threadId: string }).threadId,
421 + );
422 + assert.equal(new Set(ids).size, 3);
423 + const read = (threadId: unknown) =>
424 + t3Default(sample, "t3_thread_read", { threadId }) as {
425 + thread: { title: string; status: string; createdBy?: string };
426 + items: Array<{ text: string; createdBy: string; creationSource: string }>;
427 + };
428 + const idle = t3Default(sample, "create_threads", { threads: [{ title: "Later" }] }) as {
429 + threads: Array<{ threadId: string; status: string; runId?: string }>;
430 + };
431 + assert.equal(idle.threads[0]?.status, "idle");
432 + assert.isUndefined(idle.threads[0]?.runId);
433 + assert.equal(read(idle.threads[0]?.threadId).thread.status, "idle");
434 + assert.deepEqual(read(idle.threads[0]?.threadId).items, []);
435 + assert.equal(read(ids[1]).thread.title, "CSV");
436 + assert.equal(read(ids[2]).items[0]?.text, "Fix admin expiry");
437 + assert.equal(read(ids[2]).thread.createdBy, "agent");
438 + assert.deepEqual(
439 + [read(ids[2]).items[0]?.createdBy, read(ids[2]).items[0]?.creationSource],
440 + ["agent", "mcp"],
441 + );
442 + assert.equal(
443 + read("11111111-1111-4111-8111-111111111111").thread.title,
444 + "Fix the flaky reconnect test",
445 + );
446 +});
447 +
448 it("does not record a deferred tool called before it was loaded", () => {
449 const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
450 const sample = makeSample(childCase, base);
evals/mcp-tool-use/run.ts
+93 -2
@@ -162,6 +162,11 @@ const McpToolUseExpect = Schema.Struct({
162 order: Schema.optional(Schema.Array(Schema.String)),
163 minCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
164 maxCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
165 + /**
166 + * One `args`-style value per new top-level thread the agent asks for, by `t3_thread_launch` or a
167 + * `create_threads` entry. Each matches a different thread's request; no other thread is asked for.
168 + */
169 + newThreads: Schema.optional(Schema.Array(Schema.Unknown)),
170 /** The last step with tool calls includes this tool, then the turn ends. */
171 endsTurnAfter: Schema.optional(Schema.String),
172 /** Regexes that no shell command may match. */
@@ -732,7 +737,8 @@ const createdPullRequests = ({ tool, input, output }: ShellEntry) =>
737 : [];
738 /**
739 * One sample's state. ToolSearch adds loaded schemas to `tools`, like Claude Code. `shell` is the
735 - * history the simulator keeps consistent with; `linked` and `nextTask` do the same for T3 results.
740 + * history the simulator keeps consistent with; `linked`, `nextTask` and `threads` do the same for
741 + * T3 results.
742 */
743 type Sample = {
744 readonly testCase: McpToolUseCase;
@@ -744,6 +750,8 @@ type Sample = {
750 readonly linked: Set<string>;
751 readonly codeStore: Record<string, unknown>;
752 nextTask: number;
753 + /** Threads the agent launched in this sample, by id, so reading one shows its own brief. */
754 + readonly threads: Map<string, { readonly title: string; readonly brief: string }>;
755 /** T3's device tools, once `withDevices` builds them for a world with devices. */
756 devices?: DeviceTools;
757 /** T3's results for prior device calls whose case result is "default". */
@@ -788,6 +796,7 @@ export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => {
796 linked: new Set([...urls, ...shell.flatMap(createdPullRequests)]),
797 codeStore: {},
798 nextTask: Math.max(0, ...taskNumbers) + 1,
799 + threads: new Map(),
800 priorResults: new Map(),
801 };
802 };
@@ -1000,9 +1009,41 @@ const linkedEntry = (sample: Sample, url: string) => {
1009 };
1010
1011 /** A believable T3 result for a tool the case does not mock. */
1003 -const t3Default = (sample: Sample, name: string, input: Readonly<Record<string, unknown>>) => {
1012 +export const t3Default = (
1013 + sample: Sample,
1014 + name: string,
1015 + input: Readonly<Record<string, unknown>>,
1016 +) => {
1017 const pr = () => pullRequest(sample.repo, input);
1018 + const newThread = (request: unknown, brief: unknown) => {
1019 + const n = sample.nextTask++;
1020 + const title =
1021 + isRecord(request) && typeof request.title === "string" ? request.title : "New thread";
1022 + const text = typeof brief === "string" ? brief.trim() : "";
1023 + sample.threads.set(uuid(n), { title, brief: text });
1024 + return {
1025 + threadId: uuid(n),
1026 + title,
1027 + ...(text ? { runId: `run-${n}`, status: "starting" } : { status: "idle" }),
1028 + };
1029 + };
1030 switch (name) {
1031 + case "t3_thread_launch":
1032 + return newThread(input, input.message);
1033 + case "create_threads":
1034 + return {
1035 + threads: (Array.isArray(input.threads) ? input.threads : []).map((request) => {
1036 + const target = isRecord(request) && isRecord(request.target) ? request.target : {};
1037 + return {
1038 + ...newThread(request, isRecord(request) ? request.prompt : undefined),
1039 + createdBy: "agent",
1040 + creationSource: "mcp",
1041 + providerInstanceId:
1042 + target.providerInstanceId ?? (sample.base.codex ? "codex" : "claudeAgent"),
1043 + model: target.model ?? sample.base.model,
1044 + };
1045 + }),
1046 + };
1047 case "orchestrator_capabilities":
1048 return {
1049 inheritedProviderInstanceId: "claudeAgent",
@@ -1108,6 +1149,20 @@ const t3Default = (sample: Sample, name: string, input: Readonly<Record<string,
1149 };
1150 case "t3_thread_read": {
1151 const threadId = valueText(input.threadId);
1152 + const launched = sample.threads.get(threadId);
1153 + if (launched !== undefined) {
1154 + const items = launched.brief
1155 + ? [{ position: 1, createdBy: "agent", creationSource: "mcp", text: launched.brief }]
1156 + : [];
1157 + const thread = {
1158 + threadId,
1159 + title: launched.title,
1160 + createdBy: "agent",
1161 + creationSource: "mcp",
1162 + status: launched.brief ? "running" : "idle",
1163 + };
1164 + return { thread, items, nextPosition: items.length + 1, hasMore: false };
1165 + }
1166 const { title, status, items } = DEFAULT_THREAD;
1167 return { thread: { threadId, title, status }, items, nextPosition: 3, hasMore: false };
1168 }
@@ -1438,6 +1493,35 @@ const matches = (expected: unknown, actual: unknown): boolean => {
1493 return Object.entries(expected).every(([path, value]) => matches(value, at(path)));
1494 };
1495
1496 +/** Each new top-level thread request in a run; a failed call created nothing. */
1497 +const newThreadRequests = (calls: ReadonlyArray<Call>) =>
1498 + calls.flatMap((call): ReadonlyArray<{ request: unknown; brief: unknown }> => {
1499 + if (call.failed === true || !isRecord(call.input)) return [];
1500 + if (call.tool === "t3_thread_launch")
1501 + return [{ request: call.input, brief: call.input.message }];
1502 + if (call.tool !== "create_threads" || !Array.isArray(call.input.threads)) return [];
1503 + return call.input.threads.map((request: unknown) => ({
1504 + request,
1505 + brief: isRecord(request) ? request.prompt : undefined,
1506 + }));
1507 + });
1508 +
1509 +/** Whether every expected value can take a different request it matches. */
1510 +const matchesDistinct = (
1511 + expected: ReadonlyArray<unknown>,
1512 + requests: ReturnType<typeof newThreadRequests>,
1513 + taken: ReadonlySet<number> = new Set(),
1514 +): boolean => {
1515 + const [first, ...rest] = expected;
1516 + if (expected.length === 0) return true;
1517 + return requests.some(
1518 + ({ request, brief }, i) =>
1519 + !taken.has(i) &&
1520 + matches(first, typeof first === "string" ? brief : request) &&
1521 + matchesDistinct(rest, requests, new Set([...taken, i])),
1522 + );
1523 +};
1524 +
1525 /** delegate_task takes `target.options` as an object or `[{id, value}]`. Grade both as an object. */
1526 const withOptionsObject = (input: unknown) => {
1527 if (!isRecord(input) || !isRecord(input.target) || !Array.isArray(input.target.options)) {
@@ -1472,6 +1556,13 @@ export const gradeSample = (expect: McpToolUseCase["expect"], run: SampleRun) =>
1556 add(`minCalls ${t} >= ${n}`, count(t) >= n);
1557 for (const [t, n] of Object.entries(expect.maxCalls ?? {}))
1558 add(`maxCalls ${t} <= ${n}`, count(t) <= n);
1559 + if (expect.newThreads !== undefined) {
1560 + const requests = newThreadRequests(calls);
1561 + add(
1562 + `newThreads ${valueText(expect.newThreads)}`,
1563 + requests.length === expect.newThreads.length && matchesDistinct(expect.newThreads, requests),
1564 + );
1565 + }
1566 if (expect.endsTurnAfter !== undefined) {
1567 const lastStep = Math.max(-1, ...calls.map((call) => call.step));
1568 const last = calls.filter((call) => call.step === lastStep).map((call) => call.tool);