@devries-cameron20 / t3code Mirror · read-only / commits / 613d1ac4f9
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 18:30 UTC

fix(mcp): background wait fails when none of its task IDs exist (#29773)

Co-authored-by: Taylor Dolezal <117500+onlydole@users.noreply.github.com> Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Taylor Dolezal committed Nov 12, 2026 at 14:56 UTC 613d1ac4f9a965e5f51b5821a1a34069e25153fd
8 files changed +276 -10
apps/server/src/mcp/toolkits/orchestrator/handlers.ts
+4 -1
@@ -75,7 +75,10 @@ const handlers = {
75 Effect.mapError(
76 (cause) =>
77 new OrchestratorMcpFailure({
78 - code: cause.reason === "unsupported" ? "invalid_request" : "orchestration_error",
78 + code:
79 + cause.reason === "unsupported" || cause.reason === "unmatched"
80 + ? "invalid_request"
81 + : "orchestration_error",
82 message: cause.message,
83 }),
84 ),
apps/server/src/orchestration-v2/ThreadManagementService.test.ts
+20 -2
@@ -462,7 +462,13 @@ it.effect("waitForThread returns the cut run once its continuation is declined",
462 }),
463 );
464
465 -it.effect.each(["live", "no-session", "unsupported-runtime"] as const)(
465 +it.effect.each([
466 + "live",
467 + "no-match",
468 + "no-match-clear",
469 + "no-session",
470 + "unsupported-runtime",
471 +] as const)(
472 "marks background work awaited on the thread's active provider thread with state %s",
473 (state) =>
474 Effect.gen(function* () {
@@ -538,7 +544,11 @@ it.effect.each(["live", "no-session", "unsupported-runtime"] as const)(
544 const result = yield* Effect.gen(function* () {
545 const service = yield* ThreadManagementService.ThreadManagementService;
546 return yield* service
541 - .awaitBackgroundTasks({ threadId, taskIds: ["tests", "finished"], awaited: true })
547 + .awaitBackgroundTasks({
548 + threadId,
549 + taskIds: state.startsWith("no-match") ? ["finished"] : ["tests", "finished"],
550 + awaited: state !== "no-match-clear",
551 + })
552 .pipe(Effect.result);
553 }).pipe(Effect.provide(Layer.merge(layerTest, ThreadCommandExecutor.layer)));
554
@@ -547,6 +557,14 @@ it.effect.each(["live", "no-session", "unsupported-runtime"] as const)(
557 success: { matched: ["tests"], unmatched: ["finished"] },
558 });
559 expect(calls).toEqual([{ providerThreadId, awaited: true }]);
560 + } else if (state === "no-match") {
561 + expect(result).toMatchObject({
562 + failure: { _tag: "ThreadManagementBackgroundWaitError", reason: "unmatched" },
563 + });
564 + expect(calls).toEqual([{ providerThreadId, awaited: true }]);
565 + } else if (state === "no-match-clear") {
566 + expect(result).toMatchObject({ success: { matched: [], unmatched: ["finished"] } });
567 + expect(calls).toEqual([{ providerThreadId, awaited: false }]);
568 } else {
569 expect(result).toMatchObject({
570 failure: {
apps/server/src/orchestration-v2/ThreadManagementService.ts
+9 -1
@@ -332,7 +332,7 @@ export class ThreadManagementBackgroundWaitError extends Schema.TaggedError<Thre
332 "ThreadManagementBackgroundWaitError",
333 {
334 threadId: ThreadId,
335 - reason: Schema.Literals(["unsupported", "unavailable", "failed"]),
335 + reason: Schema.Literals(["unsupported", "unavailable", "failed", "unmatched"]),
336 cause: Schema.optional(Schema.Defect()),
337 },
338 ) {
@@ -344,6 +344,8 @@ export class ThreadManagementBackgroundWaitError extends Schema.TaggedError<Thre
344 return "This thread has no live provider session.";
345 case "failed":
346 return "Failed to update the awaited background work.";
347 + case "unmatched":
348 + return "None of these IDs is running background work on this thread, so this request registers no wait for them. If the task already finished, read its output now. Otherwise pass the task ID of a background command or Monitor you started. Subagent, Agent tool, and delegate_task IDs don't go here.";
349 }
350 }
351 }
@@ -1503,6 +1505,12 @@ const make = Effect.gen(function* () {
1505 }),
1506 ),
1507 );
1508 + if (input.awaited && matched.length === 0) {
1509 + return yield* new ThreadManagementBackgroundWaitError({
1510 + threadId: input.threadId,
1511 + reason: "unmatched",
1512 + });
1513 + }
1514 const matchedIds = new Set(matched);
1515 return { matched, unmatched: input.taskIds.filter((taskId) => !matchedIds.has(taskId)) };
1516 });
evals/mcp-tool-use/README.md
+1 -1
@@ -26,7 +26,7 @@ Cases are JSON lines in `cases/`, one file per area. The case schema in `run.ts`
26
27 - `kind`: `bad` reproduces a mistake, `guard` protects correct behavior, `control` is ordinary work.
28 - `experience`: the real mistake, in a sentence. `source`: where it happened.
29 -- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results.
29 +- `setup`: the fake thread, its `world` (repo, branch, files, PRs, pages), `priorTurns`, and `mocks` for tool results. With `world.backgroundTasks`, `t3_background_wait` answers from T3's own handler and ThreadManagementService, so its result and errors are the checkout's; a prior wait with `"result": "default"` gets that result too.
30 - `setup.priorTurns[].toolCalls[].resultFile`: load a repository-relative file as the tool result instead of copying its contents into the case. An empty `prompt` resumes a top-level thread after its last prior tool result without adding a user message. Side-chat and subagent setups require a nonempty prompt.
31 - `expect`: the checks, such as `calls`, `callsMustNot`, `maxCalls`, `order`, `shellMust`, `textMust`, `finalTextMust` (the last text, once the turn ended), and `linked`. A call's `args` match partially: `~text` is a case-insensitive substring, `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths. Device cases also use `device` to check every child brief, forbid raw UI commands, bound failed device actions per user turn, and require artifact paths in completed final answers. Secrets cases use `secret` (see [Secrets cases](#secrets-cases)).
32
evals/mcp-tool-use/backgroundWait.ts new
+154
@@ -0,0 +1,154 @@
1 +// @effect-diagnostics nodeBuiltinImport:off
2 +/**
3 + * T3's real `t3_background_wait` handler and ThreadManagementService over a Claude session.
4 + * Only the adapter is the eval's: like Claude's, it matches task IDs against the session's
5 + * running background work, which the case lists in `world.backgroundTasks`.
6 + */
7 +import {
8 + EnvironmentId,
9 + type OrchestrationV2ProviderThread,
10 + type OrchestrationV2ThreadProjection,
11 + ProviderDriverKind,
12 + ProviderInstanceId,
13 + ProviderSessionId,
14 + ProviderThreadId,
15 + ThreadId,
16 +} from "@t3tools/contracts";
17 +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
18 +import * as Context from "effect/Context";
19 +import * as DateTime from "effect/DateTime";
20 +import * as Effect from "effect/Effect";
21 +import * as Layer from "effect/Layer";
22 +import * as References from "effect/References";
23 +// oxlint-disable-next-line t3code/no-raw-mcp-registration -- the eval calls the tool McpHttpServer registered and registers none.
24 +import { McpSchema, McpServer } from "effect/ai";
25 +
26 +import * as McpHttpServer from "../../apps/server/src/mcp/McpHttpServer.ts";
27 +import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts";
28 +import { liveThreadShell } from "../../apps/server/src/mcp/McpToolAccess.testkit.ts";
29 +import * as OrchestratorMcpService from "../../apps/server/src/mcp/OrchestratorMcpService.ts";
30 +import * as ThreadMetadataMcpService from "../../apps/server/src/mcp/ThreadMetadataMcpService.ts";
31 +import * as OrchestratorHandlers from "../../apps/server/src/mcp/toolkits/orchestrator/handlers.ts";
32 +import { OrchestratorToolkit } from "../../apps/server/src/mcp/toolkits/orchestrator/tools.ts";
33 +import * as CommandReceiptStore from "../../apps/server/src/orchestration-v2/CommandReceiptStore.ts";
34 +import * as Orchestrator from "../../apps/server/src/orchestration-v2/Orchestrator.ts";
35 +import * as ProviderSessionManager from "../../apps/server/src/orchestration-v2/ProviderSessionManager.ts";
36 +import * as ThreadManagementService from "../../apps/server/src/orchestration-v2/ThreadManagementService.ts";
37 +import * as PeerForwarding from "../../apps/server/src/peer/PeerForwarding.ts";
38 +import * as PeerLinkRequests from "../../apps/server/src/peer/PeerLinkRequests.ts";
39 +import * as PeerLinks from "../../apps/server/src/peer/PeerLinks.ts";
40 +import * as RemoteDelegation from "../../apps/server/src/peer/RemoteDelegation.ts";
41 +import * as SqlitePersistence from "../../apps/server/src/persistence/Sqlite.ts";
42 +import * as ServerSettings from "../../apps/server/src/serverSettings.ts";
43 +
44 +const threadId = ThreadId.make("mcp-tool-use-eval");
45 +const providerThreadId = ProviderThreadId.make("provider-thread:mcp-tool-use-eval");
46 +const providerSessionId = ProviderSessionId.make("provider-session:mcp-tool-use-eval");
47 +const providerInstanceId = ProviderInstanceId.make("claudeAgent");
48 +const driver = ProviderDriverKind.make("claudeAgent");
49 +
50 +/** Calls T3's `t3_background_wait` with `running` as the session's running background task IDs. */
51 +export const makeBackgroundWait = Effect.fn("evaluateMcpToolUse.makeBackgroundWait")(function* (
52 + running: ReadonlyArray<string>,
53 +) {
54 + const now = yield* DateTime.now;
55 + const providerThread = {
56 + id: providerThreadId,
57 + driver,
58 + providerInstanceId,
59 + providerSessionId,
60 + appThreadId: threadId,
61 + nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
62 + pendingBackgroundTasks: running.map((taskId) => ({ taskId, kind: "command" })),
63 + createdAt: now,
64 + updatedAt: now,
65 + } as unknown as OrchestrationV2ProviderThread;
66 + const runtime = {
67 + driver,
68 + providerSessionId,
69 + awaitBackgroundTasks: (input: { readonly taskIds: ReadonlyArray<string> }) =>
70 + Effect.succeed({ matched: [...new Set(input.taskIds)].filter((id) => running.includes(id)) }),
71 + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime;
72 + const real = yield* Layer.build(
73 + ThreadManagementService.layer.pipe(
74 + Layer.provide(
75 + Layer.mergeAll(
76 + Layer.mock(Orchestrator.OrchestratorV2)({
77 + getThreadRecords: () =>
78 + Effect.succeed({
79 + thread: { id: threadId, deletedAt: null, activeProviderThreadId: providerThreadId },
80 + providerThreads: [providerThread],
81 + } as unknown as OrchestrationV2ThreadProjection),
82 + }),
83 + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
84 + get: () => Effect.succeedSome(runtime),
85 + }),
86 + ServerSettings.layerTest(),
87 + CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)),
88 + ),
89 + ),
90 + ),
91 + // The receipt store's in-memory migrations log every step.
92 + ).pipe(Effect.provideService(References.MinimumLogLevel, "Error"));
93 + const service = Context.get(real, ThreadManagementService.ThreadManagementService);
94 + // The caller's thread is live on the Claude instance, mid-turn.
95 + const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({
96 + ...service,
97 + getThreadShell: () => Effect.succeed({ ...liveThreadShell(threadId), providerInstanceId }),
98 + });
99 + const context = yield* Layer.build(
100 + McpHttpServer.toolkitRegistration(OrchestratorToolkit, OrchestratorHandlers.layer).pipe(
101 + Layer.provideMerge(McpHttpServer.layerInProcessMcpServer),
102 + Layer.provide(Layer.mock(OrchestratorMcpService.OrchestratorMcpService)({})),
103 + Layer.provide(Layer.mock(ThreadMetadataMcpService.ThreadMetadataMcpService)({})),
104 + Layer.provide(threads),
105 + Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})),
106 + Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})),
107 + Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})),
108 + Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})),
109 + ),
110 + );
111 + type Entry = Parameters<McpServer.McpServer["Service"]["addTool"]>[0];
112 + // The registry keeps each tool's handler; its public type lists only definitions.
113 + const tools = Context.get(context, McpServer.McpServer).tools as ReadonlyArray<Entry>;
114 + const entry = tools.find((tool) => tool.tool.name === "t3_background_wait");
115 + if (entry === undefined) return yield* Effect.die(new Error("T3 has no t3_background_wait."));
116 + const scope: McpInvocationContext.McpThreadInvocationScope = {
117 + environmentId: EnvironmentId.make("00000000-0000-4000-8000-0000000000e0"),
118 + requestNamespace: "mcp-tool-use-eval",
119 + thread: { threadId, providerSessionId: "mcp-tool-use-eval", providerInstanceId },
120 + client: undefined,
121 + capabilities: new Set<McpInvocationContext.McpCapability>(["orchestration"]),
122 + issuedAt: 0,
123 + };
124 + const request = McpSchema.McpRequestContext.of({
125 + clientId: 0,
126 + protocolVersion: "2025-06-18",
127 + clientCapabilities: {},
128 + });
129 + return Effect.fn("evaluateMcpToolUse.backgroundWait")(function* (input: unknown) {
130 + const result = yield* entry.handle(input).pipe(
131 + Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
132 + Effect.provideService(McpSchema.McpRequestContext, request),
133 + // An MCP client shows a protocol error, such as bad parameters, as the tool's error.
134 + Effect.catch((error) =>
135 + Effect.succeed(
136 + new McpSchema.CallToolResult({
137 + isError: true,
138 + content: [{ type: "text", text: error.message }],
139 + }),
140 + ),
141 + ),
142 + );
143 + if (!("content" in result))
144 + return yield* Effect.die(new Error("Background wait asked for input."));
145 + const text = result.content
146 + .flatMap((part) => (part.type === "text" ? [part.text] : []))
147 + .join("\n");
148 + if (text === "Tool execution failed due to an internal server error.") {
149 + return yield* Effect.die(new Error("t3_background_wait hit a defect in T3 or the eval."));
150 + }
151 + return { text, isError: result.isError === true };
152 + });
153 +});
154 +export type BackgroundWait = Effect.Success<ReturnType<typeof makeBackgroundWait>>;
evals/mcp-tool-use/cases/orchestration.ndjson
+2
@@ -34,6 +34,8 @@
34 {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."}
35 {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."}
36 {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}}
37 +{"id": "background-wait-unmatched-id", "area": "orchestration", "kind": "guard", "pattern": "waits on a task ID that matches no background work, then ends the turn as if waiting", "experience": "One agent called t3_background_wait with the literal ID 'placeholder', another with a Claude subagent ID. The tool returned a normal result listing the ID under `unmatched`, so the agent said it would continue when the work came back and ended its turn. Nothing ever woke it.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["placeholder"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"calls": [{"tool": "t3_background_wait", "args": {"taskIds": ["bq7m2x9k"]}}]}, "source": {"issue": "#29772"}, "notes": "The seeded wait gets T3's real result for an ID the session does not know. The tests are still running, so the only way to keep the thread waiting is a wait on bq7m2x9k. In replay, Sonnet 5.5 and Opus 5.5 also recover from main's success result, so this guards the recovery rather than reproducing the incident. Claude only: t3_background_wait is a Claude tool.", "maxSteps": 8}
38 +{"id": "background-wait-matched-id", "area": "orchestration", "kind": "control", "pattern": "waits on the right task ID, then ends the turn", "experience": "With the background command's own ID registered, ending the turn is correct: T3 keeps the thread working and wakes it when the tests finish.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["bq7m2x9k"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"maxCalls": {"t3_background_wait": 1, "Bash": 3}, "notCalls": ["ScheduleWakeup"]}, "source": {"issue": "#29772"}, "notes": "Same thread as background-wait-unmatched-id, with a wait T3 matched. Guards against a fix that makes agents distrust a good wait and poll; a few looks at the output are ordinary. Claude only.", "maxSteps": 8}
39 {"id": "settle-this-thread-after-cleanup", "area": "orchestration", "kind": "bad", "pattern": "settle read as plain English for finish", "experience": "Asked to clean up scratch files and settle this thread, the agent deleted the files and ended the turn without calling t3_thread_organize (#26749).", "rule": "A request to settle this thread calls t3_thread_organize with action settle.", "privacy": "commit-safe", "notes": "From #26749. Claude defers MCP tools, so only the tool's name is visible until ToolSearch loads it; 'settle' appears only in the t3_thread_organize description.", "setup": {"role": "top", "world": {"files": {"scratch/run-1.log": "ok\n", "scratch/run-2.log": "ok\n", "src/index.ts": "export const x = 1;\n"}}, "mocks": {"t3_thread_organize": {"settlesWhenTurnEnds": true}}}, "prompt": "Clean up the scratch files and settle this thread.", "expect": {"calls": [{"tool": "t3_thread_organize", "args": {"action": "settle"}}], "notCalls": ["t3_thread_update"]}}
40 {"id": "settle-this-thread-bare", "area": "orchestration", "kind": "bad", "pattern": "settle read as plain English for finish", "experience": "After a finished Q&A, asked only to settle this thread, the agent replied in text and ended the turn without calling t3_thread_organize (#26749).", "rule": "A request to settle this thread calls t3_thread_organize with action settle.", "privacy": "commit-safe", "notes": "From #26749. The prior turn is general knowledge with no claims about the repo, so the only work left is settling.", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "In general, what's the difference between an HTTP 401 and a 403?"}, {"role": "assistant", "text": "A 401 means the request has no valid credentials, so the client should authenticate and try again. A 403 means the server knows who you are but won't let you do this, so retrying with the same credentials won't help."}], "mocks": {"t3_thread_organize": {"settlesWhenTurnEnds": true}}}, "prompt": "Makes sense, thanks. Settle this thread.", "expect": {"calls": [{"tool": "t3_thread_organize", "args": {"action": "settle"}}], "notCalls": ["t3_thread_update"]}}
41 {"id": "settle-on-a-choice-is-not-organize", "area": "orchestration", "kind": "guard", "pattern": "settle read as plain English for finish", "experience": "Guard for the #26749 instruction: 'settle on' a choice is ordinary English and must not settle the thread.", "rule": "Only a request to settle a thread calls t3_thread_organize.", "privacy": "commit-safe", "notes": "Catches over-triggering on the settle bullet in the orchestration instructions.", "setup": {"role": "top", "world": {"files": {"src/db.ts": "export const driver = process.env.DB_DRIVER ?? 'sqlite';\n"}}}, "prompt": "We keep going back and forth on the database for this thread. Let's settle on Postgres: make it the default driver in src/db.ts.", "expect": {"notCalls": ["t3_thread_organize"]}}
evals/mcp-tool-use/run.test.ts
+37
@@ -39,6 +39,7 @@ import {
39 t3Default,
40 t3SystemAppend,
41 t3SubagentContext,
42 + withBackgroundWait,
43 withDevices,
44 } from "./run.ts";
45 import { briefRoute, gradeDevice, shellInvocations } from "./device-grading.ts";
@@ -1146,6 +1147,42 @@ it.effect("builds device cases from product-backed worlds and synthetic prior tu
1147 }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1148 );
1149
1150 +// Models recover from either result, so this keeps the guard case failing on the old success.
1151 +it.effect("seeds T3's error for a prior background wait whose IDs match nothing", () =>
1152 + Effect.gen(function* () {
1153 + const cases = NodeFS.readFileSync(
1154 + new URL("./cases/orchestration.ndjson", import.meta.url),
1155 + "utf8",
1156 + )
1157 + .trim()
1158 + .split("\n")
1159 + .map((line) => decodeDeviceCase(line));
1160 + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false });
1161 + for (const [id, isError] of [
1162 + ["background-wait-unmatched-id", true],
1163 + ["background-wait-matched-id", false],
1164 + ] as const) {
1165 + const testCase = cases.find((c) => c.id === id)!;
1166 + const [wait] = (testCase.setup.priorTurns ?? [])
1167 + .flatMap((turn) => turn.toolCalls ?? [])
1168 + .filter((call) => call.tool === "t3_background_wait");
1169 + assert.isDefined(wait, id);
1170 + const sample = makeSample(testCase, base);
1171 + yield* withBackgroundWait(sample);
1172 + assert.equal(sample.priorErrors.has(wait), isError, id);
1173 + const blocks = buildMessages(sample).flatMap((message) => message.content);
1174 + const waitUse = blocks.find(
1175 + (block) => block.type === "tool_use" && String(block.name).endsWith("t3_background_wait"),
1176 + );
1177 + const replayed = blocks.find(
1178 + (block) => block.type === "tool_result" && block.tool_use_id === waitUse?.id,
1179 + );
1180 + assert.isDefined(replayed, id);
1181 + assert.equal(replayed?.is_error === true, isError, id);
1182 + }
1183 + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1184 +);
1185 +
1186 const secretCases = () =>
1187 NodeFS.readFileSync(new URL("./cases/secrets.ndjson", import.meta.url), "utf8")
1188 .trim()
evals/mcp-tool-use/run.ts
+49 -5
@@ -72,6 +72,7 @@ import {
72 } from "../../apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts";
73 import { proxyConfig } from "../shared/proxy.ts";
74 import { changeRequestsCreatedBy } from "../../apps/server/src/orchestration-v2/PullRequestSyncReactor.ts";
75 +import { type BackgroundWait, makeBackgroundWait } from "./backgroundWait.ts";
76 import { DeviceExpect, gradeDevice } from "./device-grading.ts";
77 import { FileLinksExpect, gradeFileLinks } from "./file-link-grading.ts";
78 import {
@@ -168,7 +169,10 @@ const Rest = [Schema.Record(Schema.String, Schema.Unknown)] as const;
169 const PriorToolCall = Schema.Struct({
170 tool: Schema.String,
171 input: Schema.Unknown,
171 - /** For a device tool, `"default"` runs the call on the world's devices for T3's result. */
172 + /**
173 + * For a device tool or t3_background_wait, `"default"` runs the call on the world's devices or
174 + * background tasks for T3's result.
175 + */
176 result: Schema.Unknown,
177 /** Repository-relative file whose current contents supply this tool result. */
178 resultFile: Schema.optional(Schema.String),
@@ -273,6 +277,11 @@ const McpToolUseWorld = Schema.StructWithRest(
277 deviceInvalidCapture: Schema.optional(Schema.Boolean),
278 /** Devices another thread's agent-device session holds, by device id: that thread's id. */
279 deviceOwners: Schema.optional(Schema.Record(Schema.String, Schema.String)),
280 + /**
281 + * Task IDs of the background commands and Monitors still running on the agent's Claude
282 + * session. With them, T3's own t3_background_wait handler answers from this world.
283 + */
284 + backgroundTasks: Schema.optional(Schema.Array(Schema.String)),
285 /** PRs already linked to the thread, with the state T3 last saw. */
286 linkedPullRequests: Schema.optional(
287 Schema.Array(
@@ -860,6 +869,10 @@ type Sample = {
869 devices?: DeviceTools;
870 /** T3's results for prior device calls whose case result is "default". */
871 readonly priorResults: Map<unknown, string>;
872 + /** Prior calls whose T3 result is an error, so the agent sees it as one. */
873 + readonly priorErrors: Set<unknown>;
874 + /** T3's t3_background_wait, once `withBackgroundWait` builds it for `world.backgroundTasks`. */
875 + backgroundWait?: BackgroundWait;
876 /** Calls so far per mocked tool, for mocks that answer in sequence. */
877 readonly mockCalls: Map<string, number>;
878 };
@@ -906,6 +919,7 @@ export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => {
919 secretFiles: [],
920 secretRequests: 0,
921 priorResults: new Map(),
922 + priorErrors: new Set(),
923 mockCalls: new Map(),
924 };
925 };
@@ -937,6 +951,25 @@ export const withDevices = Effect.fn("evaluateMcpToolUse.withDevices")(function*
951 }
952 });
953
954 +/**
955 + * Builds T3's t3_background_wait for a world with background tasks, and answers prior waits whose
956 + * result is "default" with T3's result.
957 + */
958 +export const withBackgroundWait = Effect.fn("evaluateMcpToolUse.withBackgroundWait")(function* (
959 + sample: Sample,
960 +) {
961 + const running = sample.testCase.setup.world?.backgroundTasks;
962 + if (running === undefined) return;
963 + const backgroundWait = yield* makeBackgroundWait(running);
964 + sample.backgroundWait = backgroundWait;
965 + for (const call of (sample.testCase.setup.priorTurns ?? []).flatMap((t) => t.toolCalls ?? [])) {
966 + if (call.result !== "default" || call.tool !== "t3_background_wait") continue;
967 + const result = yield* backgroundWait(call.input);
968 + sample.priorResults.set(call, result.text);
969 + if (result.isError) sample.priorErrors.add(call);
970 + }
971 +});
972 +
973 // Main's history under every case's world.
974 const MAIN_COMMITS = [
975 { sha: "9f2c1ab", subject: "feat(web): show the model name in the thread header" },
@@ -1024,8 +1057,13 @@ export const buildMessages = (sample: Sample) => {
1057 const calls = (turn.toolCalls ?? []).map((call) => {
1058 const tool = findTool(base, call.tool);
1059 if (tool === undefined) throw new Error(`${testCase.id}: unknown prior tool ${call.tool}.`);
1027 - if (call.result === "default" && isDeviceTool(call.tool) && !sample.priorResults.has(call)) {
1028 - throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run withDevices.`);
1060 + const builder = isDeviceTool(call.tool)
1061 + ? "withDevices"
1062 + : call.tool === "t3_background_wait"
1063 + ? "withBackgroundWait"
1064 + : undefined;
1065 + if (call.result === "default" && builder && !sample.priorResults.has(call)) {
1066 + throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run ${builder}.`);
1067 }
1068 return { ...call, id: `toolu_prior_${nextId++}`, name: tool.name, tool, prior: call };
1069 });
@@ -1047,7 +1085,8 @@ export const buildMessages = (sample: Sample) => {
1085 normalizeToolName(tool.name) === "orchestrator_capabilities"
1086 ? capabilities(sample, result)
1087 : (sample.priorResults.get(prior) ?? result);
1050 - push("user", toolResult(id, replayed));
1088 + const answer = toolResult(id, replayed);
1089 + push("user", sample.priorErrors.has(prior) ? { ...answer, is_error: true } : answer);
1090 }
1091 if (turn.text) push(turn.role, text(turn.text));
1092 }
@@ -1640,6 +1679,10 @@ export const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(fun
1679 const result = yield* sample.devices.call(name, input, failure);
1680 return result.isError ? failed(result.text) : toolResult(call.id, result.text);
1681 }
1682 + if (sample.backgroundWait && isT3Name(call.name) && name === "t3_background_wait" && fromWorld) {
1683 + const result = yield* sample.backgroundWait(input);
1684 + return result.isError ? failed(result.text) : toolResult(call.id, result.text);
1685 + }
1686 // A pasted secret is saved without a card, so a mocked card outcome does not apply.
1687 if (mock !== undefined && mock !== "default" && !pastedSecret(name, input)) {
1688 if (name === "request_secret") savedSecretFile(sample, mock);
@@ -2142,6 +2185,7 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* (
2185 ) {
2186 const sample = { ...makeSample(testCase, base), shellUsage: spent.shellUsage };
2187 yield* withDevices(sample);
2188 + yield* withBackgroundWait(sample);
2189 const runPromise = Effect.runPromiseWith(yield* Effect.context<HttpClient.HttpClient>());
2190 const messages = buildMessages(sample);
2191 const input = base.codex ? codexMessages(messages) : undefined;
@@ -2562,7 +2606,7 @@ const main = Effect.gen(function* () {
2606 // One JSON line per case. Without --case this also checks that every case builds.
2607 for (const testCase of selected) {
2608 const sample = makeSample(testCase, baseFor(testCase));
2565 - yield* Effect.scoped(withDevices(sample));
2609 + yield* Effect.scoped(Effect.andThen(withDevices(sample), withBackgroundWait(sample)));
2610 const messages = buildMessages(sample);
2611 const body = sample.base.codex
2612 ? codexRequest(sample.base, codexMessages(messages))