@devries-cameron20 / t3code-1 / commits / ed57b9b5ec
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a follow-up to a finished remote task is a new task that wakes the parent (#26776)

Julius Marminge committed Nov 11, 2026 at 15:38 UTC ed57b9b5ec7e7bbe9a25da1d8883f218eaee1b75
8 files changed +516 -30
apps/server/src/mcp/McpToolAccess.ts
+44 -2
@@ -46,6 +46,7 @@ const refuseOutsideConstruction = (token: symbol) => {
46 let declare: <P, A, E, R>(
47 handle: (params: P) => Effect.Effect<A, E, R>,
48 remote: RemoteCheck<P> | undefined,
49 + forwarded?: Forwarded<P, A, E, R>,
50 ) => Declaration<(params: P) => Effect.Effect<A, E, R>>;
51
52 /**
@@ -58,6 +59,16 @@ type RemoteCheck<P> = (
59 ) => Effect.Effect<unknown, OrchestratorMcpFailure, Effect.Services<typeof writingCaller>>;
60 let remoteCheckOf: <Handler>(declaration: Declaration<Handler>) => RemoteCheck<never> | undefined;
61
62 +/** What a call answered in a linked environment does here before it returns. */
63 +type Forwarded<P, A, E, R> = (
64 + params: P,
65 + environmentId: EnvironmentId,
66 + result: A,
67 +) => Effect.Effect<A, E, R>;
68 +let forwardedOf: <Handler>(
69 + declaration: Declaration<Handler>,
70 +) => Forwarded<never, unknown, unknown, unknown> | undefined;
71 +
72 type CheckedHandlerOf<D> = D extends Declaration<infer Handler> ? Handler : never;
73
74 interface CheckedHandler extends Struct.Lambda {
@@ -74,19 +85,29 @@ let handlersLayer: <Tools extends Record<string, Tool.Any>, EX, RX>(
85 export class Declaration<out Handler> {
86 readonly #handle: Handler;
87 readonly #remote: RemoteCheck<never> | undefined;
88 + readonly #forwarded: Forwarded<never, unknown, unknown, unknown> | undefined;
89 private constructor(
90 token: typeof builtHere,
91 handle: Handler,
92 remote: RemoteCheck<never> | undefined,
93 + forwarded: Forwarded<never, unknown, unknown, unknown> | undefined,
94 ) {
95 refuseOutsideConstruction(token);
96 this.#handle = handle;
97 this.#remote = remote;
98 + this.#forwarded = forwarded;
99 }
100 static {
87 - declare = (handle, remote) => new Declaration(builtHere, handle, remote);
101 + declare = (handle, remote, forwarded) =>
102 + new Declaration(
103 + builtHere,
104 + handle,
105 + remote,
106 + forwarded as Forwarded<never, unknown, unknown, unknown> | undefined,
107 + );
108 checkedHandler = Struct.lambda<CheckedHandler>((declaration) => declaration.#handle);
109 remoteCheckOf = (declaration) => declaration.#remote;
110 + forwardedOf = (declaration) => declaration.#forwarded;
111 }
112 }
113
@@ -281,6 +302,20 @@ export const either = <P, A, E, R, A2, E2, R2>(
302 );
303 };
304
305 +/**
306 + * `declaration`, whose result from a linked environment then goes through
307 + * `forwarded` here, for what this environment records about the call.
308 + */
309 +export const afterForwarding = <P, A, E, R, E2, R2>(
310 + declaration: Declaration<(params: P) => Effect.Effect<A, E, R>>,
311 + forwarded: Forwarded<P, A, E2, R2>,
312 +) =>
313 + declare(
314 + checkedHandler(declaration) as (params: P) => Effect.Effect<A, E | E2, R | R2>,
315 + remoteCheckOf(declaration) as RemoteCheck<P> | undefined,
316 + forwarded,
317 + );
318 +
319 const fullAccessRequired =
320 "Changing projects or environment settings needs a live full-access/default calling thread or a full-access client.";
321
@@ -387,6 +422,9 @@ const routed = Effect.fn("McpToolAccess.routed")(function* <Tools extends Record
422 const tool = toolkit.tools[name];
423 if (tool === undefined || !takesLinkedEnvironment(tool)) continue;
424 const remote = remoteCheckOf(declaration);
425 + const forwarded = forwardedOf(declaration) as
426 + | Forwarded<object, unknown, Tool.Failure<Tools[keyof Tools]>, never>
427 + | undefined;
428 if (remote === undefined)
429 return yield* Effect.die(
430 `${name} takes a linked environment, but it acts as its calling thread, which is here.`,
@@ -402,7 +440,11 @@ const routed = Effect.fn("McpToolAccess.routed")(function* <Tools extends Record
440 if (scope.linkedEnvironments === false) return yield* PeerForwarding.linkedEnvironmentsOff;
441 yield* (remote as RemoteCheck<typeof params>)(params);
442 if (Option.isNone(forwarding)) return yield* linksUnavailable;
405 - return yield* forwarding.value.call(scope, tool, target, { ...params, environmentId });
443 + const result = yield* forwarding.value.call(scope, tool, target, {
444 + ...params,
445 + environmentId,
446 + });
447 + return forwarded === undefined ? result : yield* forwarded(params, target, result);
448 });
449 }
450 return handlers as Toolkit.HandlersFrom<Tools>;
apps/server/src/mcp/toolkits/orchestrator/handlers.ts
+15 -5
@@ -5,6 +5,7 @@ import * as Effect from "effect/Effect";
5 import * as PeerForwarding from "../../../peer/PeerForwarding.ts";
6 import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts";
7 import * as PeerLinks from "../../../peer/PeerLinks.ts";
8 +import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts";
9 import { readCaller } from "../../threadAccess.ts";
10 import * as McpInvocationContext from "../../McpInvocationContext.ts";
11 import * as McpToolAccess from "../../McpToolAccess.ts";
@@ -168,13 +169,22 @@ const handlers = {
169 return yield* service.update(scope, input);
170 }),
171 ),
171 - t3_thread_send: McpToolAccess.writesThreads(
172 - (input) => [input.threadId],
173 - (input) =>
172 + t3_thread_send: McpToolAccess.afterForwarding(
173 + McpToolAccess.writesThreads(
174 + (input) => [input.threadId],
175 + (input) =>
176 + Effect.gen(function* () {
177 + const scope = yield* McpInvocationContext.McpInvocationContext;
178 + const service = yield* OrchestratorMcpService.OrchestratorMcpService;
179 + return yield* service.sendToThread(scope, input);
180 + }),
181 + ),
182 + // There the thread's parent is here, so only this side can make a follow-up a task.
183 + (input, environmentId, sent) =>
184 Effect.gen(function* () {
185 const scope = yield* McpInvocationContext.McpInvocationContext;
176 - const service = yield* OrchestratorMcpService.OrchestratorMcpService;
177 - return yield* service.sendToThread(scope, input);
186 + const remote = yield* RemoteDelegation.RemoteDelegation;
187 + return yield* remote.followUp(scope, environmentId, input, sent);
188 }),
189 ),
190 t3_thread_wait: McpToolAccess.reads((input) =>
apps/server/src/mcp/toolkits/orchestrator/tools.ts
+4 -1
@@ -46,6 +46,7 @@ import * as ThreadManagementService from "../../../orchestration-v2/ThreadManage
46 import * as PeerForwarding from "../../../peer/PeerForwarding.ts";
47 import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts";
48 import * as PeerLinks from "../../../peer/PeerLinks.ts";
49 +import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts";
50 import * as McpInvocationContext from "../../McpInvocationContext.ts";
51 import * as OrchestratorMcpService from "../../OrchestratorMcpService.ts";
52 import * as ThreadMetadataMcpService from "../../ThreadMetadataMcpService.ts";
@@ -63,6 +64,8 @@ const linkingDependencies = [
64 PeerLinkRequests.PeerLinkRequests,
65 PeerLinks.PeerLinks,
66 ];
67 +/** A send there may record a follow-up to a remote task. */
68 +const sendDependencies = [...dependencies, RemoteDelegation.RemoteDelegation];
69 const threadMetadataDependencies = [
70 McpInvocationContext.McpInvocationContext,
71 ThreadManagementService.ThreadManagementService,
@@ -279,7 +282,7 @@ const ThreadSendTool = Tool.make("t3_thread_send", {
282 success: OrchestratorMcpThreadSendResult,
283 failure: OrchestratorMcpFailure,
284 failureMode: "return",
282 - dependencies,
285 + dependencies: sendDependencies,
286 })
287 .annotate(Tool.Title, "Send to a T3 thread")
288 .annotate(Tool.Destructive, true)
apps/server/src/orchestration-v2/CommandPolicy.ts
+10
@@ -10,6 +10,7 @@ import {
10 ProviderInstanceId,
11 ThreadId,
12 } from "@t3tools/contracts";
13 +import { latestExecutedRun } from "@t3tools/shared/orchestrationV2ThreadError";
14 import * as Context from "effect/Context";
15 import * as Effect from "effect/Effect";
16 import * as Layer from "effect/Layer";
@@ -125,6 +126,15 @@ export function isActiveRun(run: OrchestrationV2Run): boolean {
126 );
127 }
128
129 +/** The run a follow-up to a finished child binds to: the latest active one, else the last that ran. */
130 +export function followUpOwnerRun(
131 + runs: ReadonlyArray<OrchestrationV2Run>,
132 +): OrchestrationV2Run | null {
133 + return (
134 + runs.filter(isActiveRun).toSorted((a, b) => b.ordinal - a.ordinal)[0] ?? latestExecutedRun(runs)
135 + );
136 +}
137 +
138 function providerTurnCapabilities(
139 projection: OrchestrationV2ThreadProjection,
140 run: OrchestrationV2Run,
apps/server/src/orchestration-v2/Orchestrator.ts
+17 -5
@@ -107,6 +107,7 @@ import {
107 import { CheckpointServiceV2 } from "./CheckpointService.ts";
108 import {
109 CommandPolicyV2,
110 + followUpOwnerRun,
111 goalCommandStopsActiveRun,
112 isActiveRun,
113 isNativeMaintenanceCommand,
@@ -8952,9 +8953,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8953 task.id === (child.forkedFrom?.type === "node" ? child.forkedFrom.nodeId : undefined),
8954 );
8955 if (original === undefined || original.result === null) return;
8955 - const owner =
8956 - parent.runs.filter(isActiveRun).toSorted((a, b) => b.ordinal - a.ordinal)[0] ??
8957 - latestExecutedRun(parent.runs);
8956 + const owner = followUpOwnerRun(parent.runs);
8957 if (owner === null || owner.rootNodeId === null) return;
8958 const now = message.createdAt;
8959 const adapter = yield* providerAdapters.get(run.providerInstanceId);
@@ -9072,7 +9071,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9071 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9072 ) {
9073 const parentProjection = yield* projectionStore
9075 - .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts"])
9074 + .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts", "subagents"])
9075 .pipe(
9076 Effect.mapError(
9077 (cause) => new OrchestratorProjectionError({ threadId: command.parentThreadId, cause }),
@@ -9081,7 +9080,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9080 const parentRun = parentProjection.runs.find(
9081 (candidate) => candidate.id === command.parentRunId,
9082 );
9084 - if (parentRun === undefined || !isActiveRun(parentRun) || parentRun.rootNodeId === null) {
9083 + // A follow-up to a finished task there binds as one to a child here does.
9084 + const followsUp = parentProjection.subagents.some(
9085 + (task) =>
9086 + task.result !== null &&
9087 + task.remoteChild?.environmentId === command.remoteChild.environmentId &&
9088 + task.remoteChild.threadId === command.remoteChild.threadId,
9089 + );
9090 + if (
9091 + parentRun === undefined ||
9092 + (followsUp
9093 + ? parentRun.id !== followUpOwnerRun(parentProjection.runs)?.id
9094 + : !isActiveRun(parentRun)) ||
9095 + parentRun.rootNodeId === null
9096 + ) {
9097 return yield* new OrchestratorDispatchError({
9098 commandId: command.commandId,
9099 commandType: command.type,
apps/server/src/peer/PeerForwarding.test.ts
+4 -1
@@ -314,7 +314,10 @@ const makeLaptop = Effect.gen(function* () {
314 Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({}),
315 Layer.mock(SecretRequests.SecretRequests)({}),
316 ServerSettings.layerTest(),
317 - Layer.mock(RemoteDelegation.RemoteDelegation)({}),
317 + // No thread here delegated to the box, so a send there is no follow-up.
318 + Layer.mock(RemoteDelegation.RemoteDelegation)({
319 + followUp: (_scope, _environmentId, _request, sent) => Effect.succeed(sent),
320 + }),
321 Layer.mock(PeerLinkRequests.PeerLinkRequests)({}),
322 ),
323 ),
apps/server/src/peer/RemoteDelegation.test.ts
+293 -10
@@ -31,6 +31,7 @@ import * as Schedule from "effect/Schedule";
31 import * as Stream from "effect/Stream";
32 import * as SubscriptionRef from "effect/SubscriptionRef";
33 import * as Tracer from "effect/Tracer";
34 +import * as TestClock from "effect/testing/TestClock";
35 import type * as SqlClient from "effect/sql/SqlClient";
36 import { McpSchema, McpServer } from "effect/ai";
37
@@ -100,6 +101,10 @@ interface BoxThread {
101 readonly laterError: Ref.Ref<string | null>;
102 /** A later run's reply, after the task's own, as a follow-up message would leave. */
103 readonly laterReply: Ref.Ref<string | null>;
104 + /** The reply of a run queued there behind the later one; once set, the thread settles on it. */
105 + readonly queuedReply: Ref.Ref<string | null>;
106 + /** A run there restarted, which its continuation finished with this reply. */
107 + readonly restart: Ref.Ref<{ readonly of: RunId; readonly reply: string } | null>;
108 /** The model the provider there reported running the task's run. */
109 readonly executedModel: Ref.Ref<string | null>;
110 /** The run each wait asked for, in order. */
@@ -120,6 +125,8 @@ interface BoxThread {
125 /** Completed on the first interrupt there. */
126 readonly interruptedOnce: Deferred.Deferred<void>;
127 readonly finished: Deferred.Deferred<void>;
128 + /** Ends the later run a message sent there started, for a wait pinned to it. */
129 + readonly laterFinished: Deferred.Deferred<void>;
130 /** While set, reading the thread there waits: completed once a read starts, released by `releaseRead`. */
131 readonly holdRead: Ref.Ref<boolean>;
132 readonly readStarted: Deferred.Deferred<void>;
@@ -147,6 +154,8 @@ const makeBoxThread = Effect.gen(function* () {
154 error: yield* Ref.make<string | null>(null),
155 laterError: yield* Ref.make<string | null>(null),
156 laterReply: yield* Ref.make<string | null>(null),
157 + queuedReply: yield* Ref.make<string | null>(null),
158 + restart: yield* Ref.make<{ readonly of: RunId; readonly reply: string } | null>(null),
159 executedModel: yield* Ref.make<string | null>(null),
160 waitedFor: yield* Ref.make<ReadonlyArray<RunId | undefined>>([]),
161 nested: yield* Ref.make(false),
@@ -158,6 +167,7 @@ const makeBoxThread = Effect.gen(function* () {
167 runless: yield* Ref.make(false),
168 interruptedOnce: yield* Deferred.make<void>(),
169 finished: yield* Deferred.make<void>(),
170 + laterFinished: yield* Deferred.make<void>(),
171 holdRead: yield* Ref.make(false),
172 readStarted: yield* Deferred.make<void>(),
173 releaseRead: yield* Deferred.make<void>(),
@@ -208,6 +218,8 @@ const serveBox = (thread: BoxThread) => {
218
219 const boxRunId = RunId.make("run:box");
220 const boxLaterRunId = RunId.make("run:box-later");
221 +const boxQueuedRunId = RunId.make("run:box-queued");
222 +const continuationOf = (runId: RunId) => RunId.make(`${runId}:restarted`);
223
224 const boxToolkitLayer = (thread: BoxThread) => {
225 const runId = boxRunId;
@@ -278,6 +290,37 @@ const boxToolkitLayer = (thread: BoxThread) => {
290 threadId: input.threadId,
291 });
292 }
293 + // The queued run's own task stays open while the follow-up ahead of it settles.
294 + if (input.runId === boxQueuedRunId) return yield* Effect.never;
295 + if (input.runId === boxLaterRunId) {
296 + const laterDone = yield* Deferred.await(thread.laterFinished).pipe(
297 + Effect.timeoutOption(input.timeoutMs),
298 + );
299 + // Settled, the thread's answer is the run queued behind it, once there is one.
300 + const queued =
301 + Option.isSome(laterDone) &&
302 + input.untilDelegatedWorkSettled === true &&
303 + (yield* Ref.get(thread.queuedReply)) !== null;
304 + // A restarted run is reported as its continuation, as the box follows the chain.
305 + const restarted = (yield* Ref.get(thread.restart))?.of;
306 + const latest = (runId: RunId) => (runId === restarted ? continuationOf(runId) : runId);
307 + return {
308 + threadId: input.threadId,
309 + run: {
310 + id: latest(boxLaterRunId),
311 + status: Option.isSome(laterDone) ? "completed" : "running",
312 + } as OrchestrationV2Run,
313 + timedOut: Option.isNone(laterDone),
314 + ...(queued
315 + ? {
316 + resultRun: {
317 + id: latest(boxQueuedRunId),
318 + status: "completed",
319 + } as OrchestrationV2Run,
320 + }
321 + : {}),
322 + };
323 + }
324 const done = yield* Deferred.await(thread.finished).pipe(
325 Effect.timeoutOption(input.timeoutMs),
326 );
@@ -319,8 +362,10 @@ const boxToolkitLayer = (thread: BoxThread) => {
362 }
363 const reply = yield* Ref.get(thread.reply);
364 const later = yield* Ref.get(thread.laterReply);
365 + const queued = yield* Ref.get(thread.queuedReply);
366 const error = yield* Ref.get(thread.error);
367 const laterError = yield* Ref.get(thread.laterError);
368 + const restart = yield* Ref.get(thread.restart);
369 // One item per page, with errors available only in the activity view.
370 const replies = [
371 ...(reply === null
@@ -356,6 +401,26 @@ const boxToolkitLayer = (thread: BoxThread) => {
401 type: "error",
402 },
403 ]),
404 + ...(queued === null
405 + ? []
406 + : [
407 + {
408 + id: TurnItemId.make("item:queued-reply"),
409 + runId: boxQueuedRunId,
410 + text: queued,
411 + type: "assistant_message",
412 + },
413 + ]),
414 + ...(restart === null
415 + ? []
416 + : [
417 + {
418 + id: TurnItemId.make("item:restarted-reply"),
419 + runId: continuationOf(restart.of),
420 + text: restart.reply,
421 + type: "assistant_message",
422 + },
423 + ]),
424 ].map((entry, position) => ({ ...entry, position }));
425 const visible =
426 options.itemId === undefined
@@ -424,10 +489,11 @@ const boxToolkitLayer = (thread: BoxThread) => {
489 stopDelegatedTasks: (input) => recordStop(input.commandId, "delegated-tasks"),
490 sendToThread: (input) =>
491 Ref.update(thread.sent, (seen) => [...seen, input.text]).pipe(
427 - Effect.as({
428 - run: { id: boxLaterRunId, status: "running" },
429 - delivery: "started",
430 - } as never),
492 + Effect.as(
493 + (input.mode === "queue"
494 + ? { run: { id: boxQueuedRunId, status: "queued" }, delivery: "queued" }
495 + : { run: { id: boxLaterRunId, status: "running" }, delivery: "started" }) as never,
496 + ),
497 ),
498 });
499 const layerBoxLaunches = Layer.mock(ThreadLaunch.ThreadLaunchService)({
@@ -1702,13 +1768,14 @@ it.effect("stopping the parent stops follow-up work on a task there that has its
1768 result: "All green.",
1769 });
1770
1705 - // The parent sends the thread there a follow-up, which starts a run there.
1771 + // The parent sends the thread there a follow-up, which starts a run there as a new task.
1772 const sent = yield* a.call("t3_thread_send", {
1773 environmentId: box.environmentId,
1774 threadId: delegated.remoteChild.threadId,
1775 message: "Now run the slow suite too.",
1776 });
1777 expect(sent.isError, JSON.stringify(sent.content)).toBe(false);
1778 + const followUpId = (sent.structuredContent as { taskId: NodeId }).taskId;
1779 expect(yield* Ref.get(thread.sent)).toEqual(["Now run the slow suite too."]);
1780 expect(yield* Ref.get(thread.interrupted)).toBe(0);
1781
@@ -1716,9 +1783,15 @@ it.effect("stopping the parent stops follow-up work on a task there that has its
1783 threadId: parentThreadId,
1784 commandId: CommandId.make("command:stop:laptop-parent-after-result"),
1785 });
1719 - // The thread there is stopped as Stop does, and the task keeps its result.
1786 + // The thread there is stopped as Stop does, for each task, and the first keeps its result.
1787 const task = yield* stoppedThere(a, delegated.taskId);
1721 - expect(yield* Ref.get(thread.stops)).toEqual(["thread.stop", "delegated-tasks"]);
1788 + expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" });
1789 + expect((yield* Ref.get(thread.stops)).toSorted()).toEqual([
1790 + "delegated-tasks",
1791 + "delegated-tasks",
1792 + "thread.stop",
1793 + "thread.stop",
1794 + ]);
1795 expect(task).toMatchObject({ status: "completed", result: "All green." });
1796 expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]);
1797
@@ -1728,7 +1801,214 @@ it.effect("stopping the parent stops follow-up work on a task there that has its
1801 commandId: CommandId.make("command:stop:laptop-parent-again"),
1802 });
1803 expect(yield* stoppedThere(a, delegated.taskId, 2)).toMatchObject({ status: "completed" });
1731 - expect(yield* Ref.get(thread.interrupted)).toBe(2);
1804 + expect(yield* stoppedThere(a, followUpId, 1)).toMatchObject({ status: "interrupted" });
1805 + expect(yield* Ref.get(thread.interrupted)).toBe(4);
1806 + }),
1807 + ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1808 +);
1809 +
1810 +it.effect.each([
1811 + { order: "sent", queuedFirst: false },
1812 + // Concurrent sends may be recorded here in the opposite order they run there.
1813 + { order: "recorded out of order", queuedFirst: true },
1814 +])(
1815 + "a follow-up there ends with its own reply, not one queued behind it ($order)",
1816 + ({ queuedFirst }) =>
1817 + Effect.scoped(
1818 + Effect.gen(function* () {
1819 + const thread = yield* makeBoxThread;
1820 + const b = yield* serveBox(thread);
1821 + const offers = yield* SubscriptionRef.make<
1822 + ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1823 + >([]);
1824 + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1825 + const a = yield* makeLaptop(database, offers);
1826 + yield* a.remote.start();
1827 + yield* linkTo(a.links, b, "full-access");
1828 + yield* seedParent(a);
1829 + const delegated = yield* delegateToBox(a);
1830 + yield* Ref.set(thread.reply, "All green.");
1831 + yield* Ref.set(thread.status, "completed");
1832 + yield* Deferred.succeed(thread.finished, undefined);
1833 + yield* taskResult(a, delegated.taskId);
1834 +
1835 + const sendFollowUp = a.call("t3_thread_send", {
1836 + environmentId: box.environmentId,
1837 + threadId: delegated.remoteChild.threadId,
1838 + message: "Now run the slow suite too.",
1839 + });
1840 + const sendQueued = a.call("t3_thread_send", {
1841 + environmentId: box.environmentId,
1842 + threadId: delegated.remoteChild.threadId,
1843 + message: "Then the flaky suite.",
1844 + mode: "queue",
1845 + });
1846 + const [first, second] = queuedFirst
1847 + ? [sendQueued, sendFollowUp]
1848 + : [sendFollowUp, sendQueued];
1849 + const firstSent = yield* first;
1850 + yield* TestClock.adjust("1 second");
1851 + const secondSent = yield* second;
1852 + const [followUp, queued] = queuedFirst ? [secondSent, firstSent] : [firstSent, secondSent];
1853 + const followUpId = (followUp.structuredContent as { taskId: NodeId }).taskId;
1854 + expect((queued.structuredContent as { taskId?: NodeId }).taskId).toBeDefined();
1855 +
1856 + // The thread there settles on the queued run, after the follow-up's own run ended.
1857 + yield* Ref.set(thread.laterReply, "The slow suite passes too.");
1858 + yield* Ref.set(thread.queuedReply, "The flaky suite passes.");
1859 + yield* Deferred.succeed(thread.laterFinished, undefined);
1860 + expect(yield* taskResult(a, followUpId)).toMatchObject({
1861 + status: "completed",
1862 + result: "The slow suite passes too.",
1863 + });
1864 + }),
1865 + ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1866 +);
1867 +
1868 +it.effect.each([
1869 + {
1870 + restarted: "follow-up",
1871 + of: boxLaterRunId,
1872 + laterReply: "Running the slow suite…",
1873 + restartReply: "The slow suite passes too.",
1874 + queuedReply: "The flaky suite passes.",
1875 + },
1876 + {
1877 + restarted: "queued",
1878 + of: boxQueuedRunId,
1879 + laterReply: "The slow suite passes too.",
1880 + restartReply: "The flaky suite passes.",
1881 + queuedReply: "Running the flaky suite…",
1882 + },
1883 +])(
1884 + "a follow-up there ends with its own reply when the $restarted run is restarted",
1885 + ({ of, laterReply, restartReply, queuedReply }) =>
1886 + Effect.scoped(
1887 + Effect.gen(function* () {
1888 + const thread = yield* makeBoxThread;
1889 + const b = yield* serveBox(thread);
1890 + const offers = yield* SubscriptionRef.make<
1891 + ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1892 + >([]);
1893 + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1894 + const a = yield* makeLaptop(database, offers);
1895 + yield* a.remote.start();
1896 + yield* linkTo(a.links, b, "full-access");
1897 + yield* seedParent(a);
1898 + const delegated = yield* delegateToBox(a);
1899 + yield* Ref.set(thread.reply, "All green.");
1900 + yield* Ref.set(thread.status, "completed");
1901 + yield* Deferred.succeed(thread.finished, undefined);
1902 + yield* taskResult(a, delegated.taskId);
1903 +
1904 + const followUp = yield* a.call("t3_thread_send", {
1905 + environmentId: box.environmentId,
1906 + threadId: delegated.remoteChild.threadId,
1907 + message: "Now run the slow suite too.",
1908 + });
1909 + yield* a.call("t3_thread_send", {
1910 + environmentId: box.environmentId,
1911 + threadId: delegated.remoteChild.threadId,
1912 + message: "Then the flaky suite.",
1913 + mode: "queue",
1914 + });
1915 + const followUpId = (followUp.structuredContent as { taskId: NodeId }).taskId;
1916 +
1917 + // The thread there settles on the queued run, after one of the two was restarted.
1918 + yield* Ref.set(thread.laterReply, laterReply);
1919 + yield* Ref.set(thread.queuedReply, queuedReply);
1920 + yield* Ref.set(thread.restart, { of, reply: restartReply });
1921 + yield* Deferred.succeed(thread.laterFinished, undefined);
1922 + expect(yield* taskResult(a, followUpId)).toMatchObject({
1923 + status: "completed",
1924 + result: "The slow suite passes too.",
1925 + });
1926 + }),
1927 + ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1928 +);
1929 +
1930 +it.effect("a follow-up sent after the parent's own run ended is still a task", () =>
1931 + Effect.scoped(
1932 + Effect.gen(function* () {
1933 + const thread = yield* makeBoxThread;
1934 + const b = yield* serveBox(thread);
1935 + const offers = yield* SubscriptionRef.make<
1936 + ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1937 + >([]);
1938 + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1939 + const a = yield* makeLaptop(database, offers);
1940 + yield* a.remote.start();
1941 + yield* linkTo(a.links, b, "full-access");
1942 + yield* seedParent(a);
1943 + const delegated = yield* delegateToBox(a);
1944 + yield* Ref.set(thread.reply, "All green.");
1945 + yield* Ref.set(thread.status, "completed");
1946 + yield* Deferred.succeed(thread.finished, undefined);
1947 + yield* taskResult(a, delegated.taskId);
1948 +
1949 + // The parent's turn ends, while a background agent of its own still holds its credential.
1950 + const now = yield* DateTime.now;
1951 + const projection = yield* a.orchestrator.getThreadProjection(parentThreadId);
1952 + const run = projection.runs.find((candidate) => candidate.id === parentRunId)!;
1953 + yield* a.sink.write({
1954 + commandId: CommandId.make("command:seed:laptop-parent-ended"),
1955 + events: [
1956 + {
1957 + id: EventId.make("event:seed-background-agent:laptop-parent"),
1958 + type: "provider-thread.updated",
1959 + threadId: parentThreadId,
1960 + driver,
1961 + providerInstanceId: instanceId,
1962 + occurredAt: now,
1963 + payload: {
1964 + ...projection.providerThreads[0]!,
1965 + pendingBackgroundTasks: [{ taskId: "native-agent", kind: "subagent" }],
1966 + updatedAt: now,
1967 + },
1968 + },
1969 + {
1970 + id: EventId.make("event:seed-run-ended:laptop-parent"),
1971 + type: "run.updated",
1972 + threadId: parentThreadId,
1973 + runId: parentRunId,
1974 + nodeId: parentRootNode,
1975 + providerInstanceId: instanceId,
1976 + occurredAt: now,
1977 + payload: { ...run, status: "completed", completedAt: now },
1978 + },
1979 + ],
1980 + });
1981 +
1982 + const sent = yield* a.call("t3_thread_send", {
1983 + environmentId: box.environmentId,
1984 + threadId: delegated.remoteChild.threadId,
1985 + message: "Now run the slow suite too.",
1986 + });
1987 + expect(sent.isError, JSON.stringify(sent.content)).toBe(false);
1988 + const followUp = sent.structuredContent as { taskId?: NodeId };
1989 + expect(followUp.taskId, JSON.stringify(followUp)).toBeDefined();
1990 + const tasks = (yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents;
1991 + expect(tasks.find((task) => task.id === followUp.taskId)).toMatchObject({
1992 + runId: parentRunId,
1993 + status: "running",
1994 + });
1995 +
1996 + // Its run there ends with its own reply, and the parent wakes for it.
1997 + const wakes = (
1998 + seen: ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>,
1999 + ) => seen.filter((offer) => offer.threadId === parentThreadId).length;
2000 + const wokenBefore = wakes(yield* SubscriptionRef.get(offers));
2001 + yield* Ref.set(thread.laterReply, "The slow suite passes too.");
2002 + yield* Deferred.succeed(thread.laterFinished, undefined);
2003 + expect(yield* taskResult(a, followUp.taskId!)).toMatchObject({
2004 + status: "completed",
2005 + result: "The slow suite passes too.",
2006 + });
2007 + const offered = yield* SubscriptionRef.changes(offers).pipe(
2008 + Stream.filter((seen) => wakes(seen) > wokenBefore),
2009 + Stream.runHead,
2010 + );
2011 + expect(Option.isSome(offered)).toBe(true);
2012 }),
2013 ).pipe(Effect.provide(NodeHttpServer.layerTest)),
2014 );
@@ -1759,11 +2039,12 @@ it.effect("a delayed Stop acknowledgement leaves a newer Stop owed", () =>
2039 });
2040 yield* Deferred.await(thread.stopStarted);
2041 // Work starts after the first Stop landed there, but before its acknowledgement returns.
1762 - yield* a.call("t3_thread_send", {
2042 + const sent = yield* a.call("t3_thread_send", {
2043 environmentId: box.environmentId,
2044 threadId: delegated.remoteChild.threadId,
2045 message: "Run another check.",
2046 });
2047 + const followUpId = (sent.structuredContent as { taskId: NodeId }).taskId;
2048 yield* a.threads.stopDelegatedTasks({
2049 threadId: parentThreadId,
2050 commandId: CommandId.make("command:stop:delayed-second"),
@@ -1777,7 +2058,9 @@ it.effect("a delayed Stop acknowledgement leaves a newer Stop owed", () =>
2058 status: "completed",
2059 result: "All green.",
2060 });
1780 - expect(yield* Ref.get(thread.interrupted)).toBe(2);
2061 + // The follow-up the work became is stopped there too, as its own task.
2062 + expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" });
2063 + expect(yield* Ref.get(thread.interrupted)).toBe(3);
2064 expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]);
2065 }),
2066 ).pipe(Effect.provide(NodeHttpServer.layerTest)),
apps/server/src/peer/RemoteDelegation.ts
+129 -6
@@ -6,6 +6,8 @@ import {
6 type OrchestratorMcpDelegateTaskInput,
7 type OrchestratorMcpTaskCancelResult,
8 type OrchestratorMcpThreadRun,
9 + type OrchestratorMcpThreadSendInput,
10 + type OrchestratorMcpThreadSendResult,
11 type OrchestratorMcpThreadTimelineItem,
12 type OrchestrationV2RemoteTaskChild,
13 type OrchestrationV2Subagent,
@@ -17,6 +19,7 @@ import {
19 } from "@t3tools/contracts";
20 import * as Context from "effect/Context";
21 import * as SecureRandom from "effect/crypto/SecureRandom";
22 +import * as DateTime from "effect/DateTime";
23 import * as Duration from "effect/Duration";
24 import * as Effect from "effect/Effect";
25 import * as Layer from "effect/Layer";
@@ -34,6 +37,7 @@ import {
37 resolveRuntimeMode,
38 taskPrompt,
39 } from "../mcp/delegatedTaskModes.ts";
40 +import { followUpOwnerRun } from "../orchestration-v2/CommandPolicy.ts";
41 import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
42 import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts";
43 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
@@ -79,6 +83,18 @@ export class RemoteDelegation extends Context.Service<
83 };
84 },
85 ) => Effect.Effect<{ readonly taskId: NodeId }, OrchestratorMcpFailure>;
86 + /**
87 + * A `t3_thread_send` there that started or queued a run on the thread of
88 + * one of this thread's tasks that has its result: like a follow-up to a
89 + * finished child here, that run is a new task, whose id the send returns.
90 + * Any other send is returned as it came.
91 + */
92 + readonly followUp: (
93 + scope: McpInvocationContext.McpInvocationScope,
94 + environmentId: EnvironmentId,
95 + request: Pick<OrchestratorMcpThreadSendInput, "message" | "clientRequestId">,
96 + sent: OrchestratorMcpThreadSendResult,
97 + ) => Effect.Effect<OrchestratorMcpThreadSendResult, OrchestratorMcpFailure>;
98 /**
99 * Stops the task's thread there, then completes it here as cancelled. A
100 * task that already has its result keeps it; its thread there is still
@@ -359,6 +375,90 @@ const make = Effect.gen(function* () {
375 },
376 );
377
378 + const followUp: RemoteDelegation["Service"]["followUp"] = Effect.fn("RemoteDelegation.followUp")(
379 + function* (scope, environmentId, request, sent) {
380 + // A steer joins a run that is already some task's, or no one's.
381 + if (scope.thread === undefined || (sent.delivery !== "started" && sent.delivery !== "queued"))
382 + return sent;
383 + const parentThreadId = scope.thread.threadId;
384 + const parent = yield* threads
385 + .getThreadRecords(parentThreadId, ["runs", "subagents"])
386 + .pipe(Effect.orElseSucceed(() => undefined));
387 + const isThere = (task: OrchestrationV2Subagent) =>
388 + task.remoteChild?.environmentId === environmentId &&
389 + task.remoteChild.threadId === sent.threadId;
390 + const finished = parent?.subagents.find((task) => isThere(task) && task.result !== null);
391 + // Bound as a follow-up here is: a background caller's foreground run may have ended.
392 + const parentRun = parent === undefined ? null : followUpOwnerRun(parent.runs);
393 + if (
394 + finished?.remoteChild === undefined ||
395 + finished.model === null ||
396 + parentRun === null ||
397 + parentRun.rootNodeId === null
398 + )
399 + return sent;
400 + // The message id there is the send's, so a retry that reached the same send records once.
401 + const commandId = CommandId.make(
402 + `command:remote-followup:${encodeURIComponent(parentThreadId)}:${encodeURIComponent(environmentId)}:${encodeURIComponent(sent.messageId)}`,
403 + );
404 + const recorded = yield* threads
405 + .dispatch({
406 + type: "delegated_task.remote.request",
407 + commandId,
408 + parentThreadId,
409 + parentRunId: parentRun.id,
410 + parentNodeId: parentRun.rootNodeId,
411 + task: request.message,
412 + ...(finished.title === null ? {} : { title: finished.title }),
413 + driver: finished.driver,
414 + modelSelection: { instanceId: finished.providerInstanceId, model: finished.model },
415 + remoteChild: {
416 + environmentId,
417 + threadId: sent.threadId,
418 + label: finished.remoteChild.label,
419 + runId: sent.runId,
420 + },
421 + completionWake: "always",
422 + })
423 + .pipe(
424 + Effect.tapError((cause) =>
425 + Effect.logWarning("remote-delegation.followup-record-failed", {
426 + parentThreadId,
427 + cause,
428 + }),
429 + ),
430 + Effect.mapError(
431 + () =>
432 + new OrchestratorMcpFailure({
433 + code: "orchestration_error",
434 + message:
435 + request.clientRequestId === undefined
436 + ? "The message was sent, but its task was not recorded; read the thread for its answer instead of sending it again."
437 + : "The message was sent, but its task was not recorded; send it again with the same clientRequestId to record it.",
438 + }),
439 + ),
440 + );
441 + const taskEvent = recorded.storedEvents.find(
442 + (stored) => stored.event.type === "subagent.updated",
443 + );
444 + const taskId =
445 + taskEvent?.event.type === "subagent.updated"
446 + ? taskEvent.event.payload.id
447 + : // A replayed request recorded the task the first time.
448 + yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe(
449 + Effect.map(
450 + ({ subagents }) =>
451 + subagents.find((task) => isThere(task) && task.remoteChild?.runId === sent.runId)
452 + ?.id,
453 + ),
454 + Effect.orElseSucceed(() => undefined),
455 + );
456 + if (taskId === undefined) return sent;
457 + yield* followOne(parentThreadId, taskId).pipe(Effect.forkIn(followers));
458 + return { ...sent, taskId };
459 + },
460 + );
461 +
462 /**
463 * Completes `task` here with its thread there's final state. Each attempt is its own
464 * command, so a failed one never blocks the next; a report refused because the task already
@@ -563,7 +663,9 @@ const make = Effect.gen(function* () {
663 ((task.status === "cancelled" || task.status === "interrupted") &&
664 remote.stoppedThere !== true)
665 ) {
566 - const key = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`;
666 + const count = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`;
667 + // Named for its run too: a follow-up task shares the thread there and counts its own stops.
668 + const key = remote.runId === undefined ? count : `${count}:${remote.runId}`;
669 yield* stopThere(scope, remote, key).pipe(
670 // Its thread there now runs above what this side may touch, say
671 // because the parent here was lowered since; no retry changes that.
@@ -632,10 +734,31 @@ const make = Effect.gen(function* () {
734 return yield* settled(parentThreadId, taskId);
735 }
736 if (!isTerminal(waited.status)) return false;
635 - const resultRun = waited.resultRun ?? {
636 - runId: remote.runId ?? waited.runId,
637 - status: waited.status,
638 - };
737 + // A later run's answer there may be another open follow-up's, or its restart, which no run id
738 + // here names; concurrent sends may record here out of their order there. Then this task's
739 + // answer is its own run's latest restart, the one waited on.
740 + const sharedThread =
741 + waited.resultRun !== undefined &&
742 + waited.resultRun.runId !== waited.runId &&
743 + (yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe(
744 + Effect.map(({ subagents }) => {
745 + const recordedAt = subagents.find((task) => task.id === taskId)?.startedAt ?? null;
746 + return subagents.some(
747 + (other) =>
748 + other.id !== taskId &&
749 + other.remoteChild?.environmentId === remote.environmentId &&
750 + other.remoteChild.threadId === remote.threadId &&
751 + // One that settled before this was recorded ran before it there.
752 + (recordedAt === null ||
753 + other.completedAt === null ||
754 + DateTime.isGreaterThan(other.completedAt, recordedAt)),
755 + );
756 + }),
757 + ));
758 + const resultRun =
759 + waited.resultRun !== undefined && !sharedThread
760 + ? waited.resultRun
761 + : { runId: waited.runId, status: waited.status };
762 const result = yield* lastReply(scope, remote, resultRun.runId, resultRun.status === "failed");
763 yield* complete(
764 parentThreadId,
@@ -793,7 +916,7 @@ const make = Effect.gen(function* () {
916 }).pipe(Effect.ignoreCause({ log: true })),
917 );
918
796 - return RemoteDelegation.of({ delegate, cancel, start });
919 + return RemoteDelegation.of({ delegate, followUp, cancel, start });
920 });
921
922 const isTerminal = (status: string) =>