The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 21:00 UTC
fix(server): a follow-up to a finished remote task is a new task that wakes the parent (#26776)
Julius Marminge committed
Nov 11, 2026 at 15:38 UTC
ed57b9b5ec7e7bbe9a25da1d8883f218eaee1b75
8 files changed
+516
-30
apps/server/src/mcp/McpToolAccess.ts
+44
-2
@@ -46,6 +46,7 @@ const refuseOutsideConstruction = (token: symbol) => {
46
let declare: <P, A, E, R>(
47
handle: (params: P) => Effect.Effect<A, E, R>,
48
remote: RemoteCheck<P> | undefined,
49
+ forwarded?: Forwarded<P, A, E, R>,
50
) => Declaration<(params: P) => Effect.Effect<A, E, R>>;
51
52
/**
@@ -58,6 +59,16 @@ type RemoteCheck<P> = (
59
) => Effect.Effect<unknown, OrchestratorMcpFailure, Effect.Services<typeof writingCaller>>;
60
let remoteCheckOf: <Handler>(declaration: Declaration<Handler>) => RemoteCheck<never> | undefined;
61
62
+/** What a call answered in a linked environment does here before it returns. */
63
+type Forwarded<P, A, E, R> = (
64
+ params: P,
65
+ environmentId: EnvironmentId,
66
+ result: A,
67
+) => Effect.Effect<A, E, R>;
68
+let forwardedOf: <Handler>(
69
+ declaration: Declaration<Handler>,
70
+) => Forwarded<never, unknown, unknown, unknown> | undefined;
71
+
72
type CheckedHandlerOf<D> = D extends Declaration<infer Handler> ? Handler : never;
73
74
interface CheckedHandler extends Struct.Lambda {
@@ -74,19 +85,29 @@ let handlersLayer: <Tools extends Record<string, Tool.Any>, EX, RX>(
85
export class Declaration<out Handler> {
86
readonly #handle: Handler;
87
readonly #remote: RemoteCheck<never> | undefined;
88
+ readonly #forwarded: Forwarded<never, unknown, unknown, unknown> | undefined;
89
private constructor(
90
token: typeof builtHere,
91
handle: Handler,
92
remote: RemoteCheck<never> | undefined,
93
+ forwarded: Forwarded<never, unknown, unknown, unknown> | undefined,
94
) {
95
refuseOutsideConstruction(token);
96
this.#handle = handle;
97
this.#remote = remote;
98
+ this.#forwarded = forwarded;
99
}
100
static {
87
- declare = (handle, remote) => new Declaration(builtHere, handle, remote);
101
+ declare = (handle, remote, forwarded) =>
102
+ new Declaration(
103
+ builtHere,
104
+ handle,
105
+ remote,
106
+ forwarded as Forwarded<never, unknown, unknown, unknown> | undefined,
107
+ );
108
checkedHandler = Struct.lambda<CheckedHandler>((declaration) => declaration.#handle);
109
remoteCheckOf = (declaration) => declaration.#remote;
110
+ forwardedOf = (declaration) => declaration.#forwarded;
111
}
112
}
113
@@ -281,6 +302,20 @@ export const either = <P, A, E, R, A2, E2, R2>(
302
);
303
};
304
305
+/**
306
+ * `declaration`, whose result from a linked environment then goes through
307
+ * `forwarded` here, for what this environment records about the call.
308
+ */
309
+export const afterForwarding = <P, A, E, R, E2, R2>(
310
+ declaration: Declaration<(params: P) => Effect.Effect<A, E, R>>,
311
+ forwarded: Forwarded<P, A, E2, R2>,
312
+) =>
313
+ declare(
314
+ checkedHandler(declaration) as (params: P) => Effect.Effect<A, E | E2, R | R2>,
315
+ remoteCheckOf(declaration) as RemoteCheck<P> | undefined,
316
+ forwarded,
317
+ );
318
+
319
const fullAccessRequired =
320
"Changing projects or environment settings needs a live full-access/default calling thread or a full-access client.";
321
@@ -387,6 +422,9 @@ const routed = Effect.fn("McpToolAccess.routed")(function* <Tools extends Record
422
const tool = toolkit.tools[name];
423
if (tool === undefined || !takesLinkedEnvironment(tool)) continue;
424
const remote = remoteCheckOf(declaration);
425
+ const forwarded = forwardedOf(declaration) as
426
+ | Forwarded<object, unknown, Tool.Failure<Tools[keyof Tools]>, never>
427
+ | undefined;
428
if (remote === undefined)
429
return yield* Effect.die(
430
`${name} takes a linked environment, but it acts as its calling thread, which is here.`,
@@ -402,7 +440,11 @@ const routed = Effect.fn("McpToolAccess.routed")(function* <Tools extends Record
440
if (scope.linkedEnvironments === false) return yield* PeerForwarding.linkedEnvironmentsOff;
441
yield* (remote as RemoteCheck<typeof params>)(params);
442
if (Option.isNone(forwarding)) return yield* linksUnavailable;
405
- return yield* forwarding.value.call(scope, tool, target, { ...params, environmentId });
443
+ const result = yield* forwarding.value.call(scope, tool, target, {
444
+ ...params,
445
+ environmentId,
446
+ });
447
+ return forwarded === undefined ? result : yield* forwarded(params, target, result);
448
});
449
}
450
return handlers as Toolkit.HandlersFrom<Tools>;
apps/server/src/mcp/toolkits/orchestrator/handlers.ts
+15
-5
@@ -5,6 +5,7 @@ import * as Effect from "effect/Effect";
5
import * as PeerForwarding from "../../../peer/PeerForwarding.ts";
6
import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts";
7
import * as PeerLinks from "../../../peer/PeerLinks.ts";
8
+import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts";
9
import { readCaller } from "../../threadAccess.ts";
10
import * as McpInvocationContext from "../../McpInvocationContext.ts";
11
import * as McpToolAccess from "../../McpToolAccess.ts";
@@ -168,13 +169,22 @@ const handlers = {
169
return yield* service.update(scope, input);
170
}),
171
),
171
- t3_thread_send: McpToolAccess.writesThreads(
172
- (input) => [input.threadId],
173
- (input) =>
172
+ t3_thread_send: McpToolAccess.afterForwarding(
173
+ McpToolAccess.writesThreads(
174
+ (input) => [input.threadId],
175
+ (input) =>
176
+ Effect.gen(function* () {
177
+ const scope = yield* McpInvocationContext.McpInvocationContext;
178
+ const service = yield* OrchestratorMcpService.OrchestratorMcpService;
179
+ return yield* service.sendToThread(scope, input);
180
+ }),
181
+ ),
182
+ // There the thread's parent is here, so only this side can make a follow-up a task.
183
+ (input, environmentId, sent) =>
184
Effect.gen(function* () {
185
const scope = yield* McpInvocationContext.McpInvocationContext;
176
- const service = yield* OrchestratorMcpService.OrchestratorMcpService;
177
- return yield* service.sendToThread(scope, input);
186
+ const remote = yield* RemoteDelegation.RemoteDelegation;
187
+ return yield* remote.followUp(scope, environmentId, input, sent);
188
}),
189
),
190
t3_thread_wait: McpToolAccess.reads((input) =>
apps/server/src/mcp/toolkits/orchestrator/tools.ts
+4
-1
@@ -46,6 +46,7 @@ import * as ThreadManagementService from "../../../orchestration-v2/ThreadManage
46
import * as PeerForwarding from "../../../peer/PeerForwarding.ts";
47
import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts";
48
import * as PeerLinks from "../../../peer/PeerLinks.ts";
49
+import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts";
50
import * as McpInvocationContext from "../../McpInvocationContext.ts";
51
import * as OrchestratorMcpService from "../../OrchestratorMcpService.ts";
52
import * as ThreadMetadataMcpService from "../../ThreadMetadataMcpService.ts";
@@ -63,6 +64,8 @@ const linkingDependencies = [
64
PeerLinkRequests.PeerLinkRequests,
65
PeerLinks.PeerLinks,
66
];
67
+/** A send there may record a follow-up to a remote task. */
68
+const sendDependencies = [...dependencies, RemoteDelegation.RemoteDelegation];
69
const threadMetadataDependencies = [
70
McpInvocationContext.McpInvocationContext,
71
ThreadManagementService.ThreadManagementService,
@@ -279,7 +282,7 @@ const ThreadSendTool = Tool.make("t3_thread_send", {
282
success: OrchestratorMcpThreadSendResult,
283
failure: OrchestratorMcpFailure,
284
failureMode: "return",
282
- dependencies,
285
+ dependencies: sendDependencies,
286
})
287
.annotate(Tool.Title, "Send to a T3 thread")
288
.annotate(Tool.Destructive, true)
apps/server/src/orchestration-v2/CommandPolicy.ts
+10
@@ -10,6 +10,7 @@ import {
10
ProviderInstanceId,
11
ThreadId,
12
} from "@t3tools/contracts";
13
+import { latestExecutedRun } from "@t3tools/shared/orchestrationV2ThreadError";
14
import * as Context from "effect/Context";
15
import * as Effect from "effect/Effect";
16
import * as Layer from "effect/Layer";
@@ -125,6 +126,15 @@ export function isActiveRun(run: OrchestrationV2Run): boolean {
126
);
127
}
128
129
+/** The run a follow-up to a finished child binds to: the latest active one, else the last that ran. */
130
+export function followUpOwnerRun(
131
+ runs: ReadonlyArray<OrchestrationV2Run>,
132
+): OrchestrationV2Run | null {
133
+ return (
134
+ runs.filter(isActiveRun).toSorted((a, b) => b.ordinal - a.ordinal)[0] ?? latestExecutedRun(runs)
135
+ );
136
+}
137
+
138
function providerTurnCapabilities(
139
projection: OrchestrationV2ThreadProjection,
140
run: OrchestrationV2Run,
apps/server/src/orchestration-v2/Orchestrator.ts
+17
-5
@@ -107,6 +107,7 @@ import {
107
import { CheckpointServiceV2 } from "./CheckpointService.ts";
108
import {
109
CommandPolicyV2,
110
+ followUpOwnerRun,
111
goalCommandStopsActiveRun,
112
isActiveRun,
113
isNativeMaintenanceCommand,
@@ -8952,9 +8953,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8953
task.id === (child.forkedFrom?.type === "node" ? child.forkedFrom.nodeId : undefined),
8954
);
8955
if (original === undefined || original.result === null) return;
8955
- const owner =
8956
- parent.runs.filter(isActiveRun).toSorted((a, b) => b.ordinal - a.ordinal)[0] ??
8957
- latestExecutedRun(parent.runs);
8956
+ const owner = followUpOwnerRun(parent.runs);
8957
if (owner === null || owner.rootNodeId === null) return;
8958
const now = message.createdAt;
8959
const adapter = yield* providerAdapters.get(run.providerInstanceId);
@@ -9072,7 +9071,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9071
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
9072
) {
9073
const parentProjection = yield* projectionStore
9075
- .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts"])
9074
+ .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts", "subagents"])
9075
.pipe(
9076
Effect.mapError(
9077
(cause) => new OrchestratorProjectionError({ threadId: command.parentThreadId, cause }),
@@ -9081,7 +9080,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9080
const parentRun = parentProjection.runs.find(
9081
(candidate) => candidate.id === command.parentRunId,
9082
);
9084
- if (parentRun === undefined || !isActiveRun(parentRun) || parentRun.rootNodeId === null) {
9083
+ // A follow-up to a finished task there binds as one to a child here does.
9084
+ const followsUp = parentProjection.subagents.some(
9085
+ (task) =>
9086
+ task.result !== null &&
9087
+ task.remoteChild?.environmentId === command.remoteChild.environmentId &&
9088
+ task.remoteChild.threadId === command.remoteChild.threadId,
9089
+ );
9090
+ if (
9091
+ parentRun === undefined ||
9092
+ (followsUp
9093
+ ? parentRun.id !== followUpOwnerRun(parentProjection.runs)?.id
9094
+ : !isActiveRun(parentRun)) ||
9095
+ parentRun.rootNodeId === null
9096
+ ) {
9097
return yield* new OrchestratorDispatchError({
9098
commandId: command.commandId,
9099
commandType: command.type,
apps/server/src/peer/PeerForwarding.test.ts
+4
-1
@@ -314,7 +314,10 @@ const makeLaptop = Effect.gen(function* () {
314
Layer.mock(ProjectCloneTracker.ProjectCloneTracker)({}),
315
Layer.mock(SecretRequests.SecretRequests)({}),
316
ServerSettings.layerTest(),
317
- Layer.mock(RemoteDelegation.RemoteDelegation)({}),
317
+ // No thread here delegated to the box, so a send there is no follow-up.
318
+ Layer.mock(RemoteDelegation.RemoteDelegation)({
319
+ followUp: (_scope, _environmentId, _request, sent) => Effect.succeed(sent),
320
+ }),
321
Layer.mock(PeerLinkRequests.PeerLinkRequests)({}),
322
),
323
),
apps/server/src/peer/RemoteDelegation.test.ts
+293
-10
@@ -31,6 +31,7 @@ import * as Schedule from "effect/Schedule";
31
import * as Stream from "effect/Stream";
32
import * as SubscriptionRef from "effect/SubscriptionRef";
33
import * as Tracer from "effect/Tracer";
34
+import * as TestClock from "effect/testing/TestClock";
35
import type * as SqlClient from "effect/sql/SqlClient";
36
import { McpSchema, McpServer } from "effect/ai";
37
@@ -100,6 +101,10 @@ interface BoxThread {
101
readonly laterError: Ref.Ref<string | null>;
102
/** A later run's reply, after the task's own, as a follow-up message would leave. */
103
readonly laterReply: Ref.Ref<string | null>;
104
+ /** The reply of a run queued there behind the later one; once set, the thread settles on it. */
105
+ readonly queuedReply: Ref.Ref<string | null>;
106
+ /** A run there restarted, which its continuation finished with this reply. */
107
+ readonly restart: Ref.Ref<{ readonly of: RunId; readonly reply: string } | null>;
108
/** The model the provider there reported running the task's run. */
109
readonly executedModel: Ref.Ref<string | null>;
110
/** The run each wait asked for, in order. */
@@ -120,6 +125,8 @@ interface BoxThread {
125
/** Completed on the first interrupt there. */
126
readonly interruptedOnce: Deferred.Deferred<void>;
127
readonly finished: Deferred.Deferred<void>;
128
+ /** Ends the later run a message sent there started, for a wait pinned to it. */
129
+ readonly laterFinished: Deferred.Deferred<void>;
130
/** While set, reading the thread there waits: completed once a read starts, released by `releaseRead`. */
131
readonly holdRead: Ref.Ref<boolean>;
132
readonly readStarted: Deferred.Deferred<void>;
@@ -147,6 +154,8 @@ const makeBoxThread = Effect.gen(function* () {
154
error: yield* Ref.make<string | null>(null),
155
laterError: yield* Ref.make<string | null>(null),
156
laterReply: yield* Ref.make<string | null>(null),
157
+ queuedReply: yield* Ref.make<string | null>(null),
158
+ restart: yield* Ref.make<{ readonly of: RunId; readonly reply: string } | null>(null),
159
executedModel: yield* Ref.make<string | null>(null),
160
waitedFor: yield* Ref.make<ReadonlyArray<RunId | undefined>>([]),
161
nested: yield* Ref.make(false),
@@ -158,6 +167,7 @@ const makeBoxThread = Effect.gen(function* () {
167
runless: yield* Ref.make(false),
168
interruptedOnce: yield* Deferred.make<void>(),
169
finished: yield* Deferred.make<void>(),
170
+ laterFinished: yield* Deferred.make<void>(),
171
holdRead: yield* Ref.make(false),
172
readStarted: yield* Deferred.make<void>(),
173
releaseRead: yield* Deferred.make<void>(),
@@ -208,6 +218,8 @@ const serveBox = (thread: BoxThread) => {
218
219
const boxRunId = RunId.make("run:box");
220
const boxLaterRunId = RunId.make("run:box-later");
221
+const boxQueuedRunId = RunId.make("run:box-queued");
222
+const continuationOf = (runId: RunId) => RunId.make(`${runId}:restarted`);
223
224
const boxToolkitLayer = (thread: BoxThread) => {
225
const runId = boxRunId;
@@ -278,6 +290,37 @@ const boxToolkitLayer = (thread: BoxThread) => {
290
threadId: input.threadId,
291
});
292
}
293
+ // The queued run's own task stays open while the follow-up ahead of it settles.
294
+ if (input.runId === boxQueuedRunId) return yield* Effect.never;
295
+ if (input.runId === boxLaterRunId) {
296
+ const laterDone = yield* Deferred.await(thread.laterFinished).pipe(
297
+ Effect.timeoutOption(input.timeoutMs),
298
+ );
299
+ // Settled, the thread's answer is the run queued behind it, once there is one.
300
+ const queued =
301
+ Option.isSome(laterDone) &&
302
+ input.untilDelegatedWorkSettled === true &&
303
+ (yield* Ref.get(thread.queuedReply)) !== null;
304
+ // A restarted run is reported as its continuation, as the box follows the chain.
305
+ const restarted = (yield* Ref.get(thread.restart))?.of;
306
+ const latest = (runId: RunId) => (runId === restarted ? continuationOf(runId) : runId);
307
+ return {
308
+ threadId: input.threadId,
309
+ run: {
310
+ id: latest(boxLaterRunId),
311
+ status: Option.isSome(laterDone) ? "completed" : "running",
312
+ } as OrchestrationV2Run,
313
+ timedOut: Option.isNone(laterDone),
314
+ ...(queued
315
+ ? {
316
+ resultRun: {
317
+ id: latest(boxQueuedRunId),
318
+ status: "completed",
319
+ } as OrchestrationV2Run,
320
+ }
321
+ : {}),
322
+ };
323
+ }
324
const done = yield* Deferred.await(thread.finished).pipe(
325
Effect.timeoutOption(input.timeoutMs),
326
);
@@ -319,8 +362,10 @@ const boxToolkitLayer = (thread: BoxThread) => {
362
}
363
const reply = yield* Ref.get(thread.reply);
364
const later = yield* Ref.get(thread.laterReply);
365
+ const queued = yield* Ref.get(thread.queuedReply);
366
const error = yield* Ref.get(thread.error);
367
const laterError = yield* Ref.get(thread.laterError);
368
+ const restart = yield* Ref.get(thread.restart);
369
// One item per page, with errors available only in the activity view.
370
const replies = [
371
...(reply === null
@@ -356,6 +401,26 @@ const boxToolkitLayer = (thread: BoxThread) => {
401
type: "error",
402
},
403
]),
404
+ ...(queued === null
405
+ ? []
406
+ : [
407
+ {
408
+ id: TurnItemId.make("item:queued-reply"),
409
+ runId: boxQueuedRunId,
410
+ text: queued,
411
+ type: "assistant_message",
412
+ },
413
+ ]),
414
+ ...(restart === null
415
+ ? []
416
+ : [
417
+ {
418
+ id: TurnItemId.make("item:restarted-reply"),
419
+ runId: continuationOf(restart.of),
420
+ text: restart.reply,
421
+ type: "assistant_message",
422
+ },
423
+ ]),
424
].map((entry, position) => ({ ...entry, position }));
425
const visible =
426
options.itemId === undefined
@@ -424,10 +489,11 @@ const boxToolkitLayer = (thread: BoxThread) => {
489
stopDelegatedTasks: (input) => recordStop(input.commandId, "delegated-tasks"),
490
sendToThread: (input) =>
491
Ref.update(thread.sent, (seen) => [...seen, input.text]).pipe(
427
- Effect.as({
428
- run: { id: boxLaterRunId, status: "running" },
429
- delivery: "started",
430
- } as never),
492
+ Effect.as(
493
+ (input.mode === "queue"
494
+ ? { run: { id: boxQueuedRunId, status: "queued" }, delivery: "queued" }
495
+ : { run: { id: boxLaterRunId, status: "running" }, delivery: "started" }) as never,
496
+ ),
497
),
498
});
499
const layerBoxLaunches = Layer.mock(ThreadLaunch.ThreadLaunchService)({
@@ -1702,13 +1768,14 @@ it.effect("stopping the parent stops follow-up work on a task there that has its
1768
result: "All green.",
1769
});
1770
1705
- // The parent sends the thread there a follow-up, which starts a run there.
1771
+ // The parent sends the thread there a follow-up, which starts a run there as a new task.
1772
const sent = yield* a.call("t3_thread_send", {
1773
environmentId: box.environmentId,
1774
threadId: delegated.remoteChild.threadId,
1775
message: "Now run the slow suite too.",
1776
});
1777
expect(sent.isError, JSON.stringify(sent.content)).toBe(false);
1778
+ const followUpId = (sent.structuredContent as { taskId: NodeId }).taskId;
1779
expect(yield* Ref.get(thread.sent)).toEqual(["Now run the slow suite too."]);
1780
expect(yield* Ref.get(thread.interrupted)).toBe(0);
1781
@@ -1716,9 +1783,15 @@ it.effect("stopping the parent stops follow-up work on a task there that has its
1783
threadId: parentThreadId,
1784
commandId: CommandId.make("command:stop:laptop-parent-after-result"),
1785
});
1719
- // The thread there is stopped as Stop does, and the task keeps its result.
1786
+ // The thread there is stopped as Stop does, for each task, and the first keeps its result.
1787
const task = yield* stoppedThere(a, delegated.taskId);
1721
- expect(yield* Ref.get(thread.stops)).toEqual(["thread.stop", "delegated-tasks"]);
1788
+ expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" });
1789
+ expect((yield* Ref.get(thread.stops)).toSorted()).toEqual([
1790
+ "delegated-tasks",
1791
+ "delegated-tasks",
1792
+ "thread.stop",
1793
+ "thread.stop",
1794
+ ]);
1795
expect(task).toMatchObject({ status: "completed", result: "All green." });
1796
expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]);
1797
@@ -1728,7 +1801,214 @@ it.effect("stopping the parent stops follow-up work on a task there that has its
1801
commandId: CommandId.make("command:stop:laptop-parent-again"),
1802
});
1803
expect(yield* stoppedThere(a, delegated.taskId, 2)).toMatchObject({ status: "completed" });
1731
- expect(yield* Ref.get(thread.interrupted)).toBe(2);
1804
+ expect(yield* stoppedThere(a, followUpId, 1)).toMatchObject({ status: "interrupted" });
1805
+ expect(yield* Ref.get(thread.interrupted)).toBe(4);
1806
+ }),
1807
+ ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1808
+);
1809
+
1810
+it.effect.each([
1811
+ { order: "sent", queuedFirst: false },
1812
+ // Concurrent sends may be recorded here in the opposite order they run there.
1813
+ { order: "recorded out of order", queuedFirst: true },
1814
+])(
1815
+ "a follow-up there ends with its own reply, not one queued behind it ($order)",
1816
+ ({ queuedFirst }) =>
1817
+ Effect.scoped(
1818
+ Effect.gen(function* () {
1819
+ const thread = yield* makeBoxThread;
1820
+ const b = yield* serveBox(thread);
1821
+ const offers = yield* SubscriptionRef.make<
1822
+ ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1823
+ >([]);
1824
+ const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1825
+ const a = yield* makeLaptop(database, offers);
1826
+ yield* a.remote.start();
1827
+ yield* linkTo(a.links, b, "full-access");
1828
+ yield* seedParent(a);
1829
+ const delegated = yield* delegateToBox(a);
1830
+ yield* Ref.set(thread.reply, "All green.");
1831
+ yield* Ref.set(thread.status, "completed");
1832
+ yield* Deferred.succeed(thread.finished, undefined);
1833
+ yield* taskResult(a, delegated.taskId);
1834
+
1835
+ const sendFollowUp = a.call("t3_thread_send", {
1836
+ environmentId: box.environmentId,
1837
+ threadId: delegated.remoteChild.threadId,
1838
+ message: "Now run the slow suite too.",
1839
+ });
1840
+ const sendQueued = a.call("t3_thread_send", {
1841
+ environmentId: box.environmentId,
1842
+ threadId: delegated.remoteChild.threadId,
1843
+ message: "Then the flaky suite.",
1844
+ mode: "queue",
1845
+ });
1846
+ const [first, second] = queuedFirst
1847
+ ? [sendQueued, sendFollowUp]
1848
+ : [sendFollowUp, sendQueued];
1849
+ const firstSent = yield* first;
1850
+ yield* TestClock.adjust("1 second");
1851
+ const secondSent = yield* second;
1852
+ const [followUp, queued] = queuedFirst ? [secondSent, firstSent] : [firstSent, secondSent];
1853
+ const followUpId = (followUp.structuredContent as { taskId: NodeId }).taskId;
1854
+ expect((queued.structuredContent as { taskId?: NodeId }).taskId).toBeDefined();
1855
+
1856
+ // The thread there settles on the queued run, after the follow-up's own run ended.
1857
+ yield* Ref.set(thread.laterReply, "The slow suite passes too.");
1858
+ yield* Ref.set(thread.queuedReply, "The flaky suite passes.");
1859
+ yield* Deferred.succeed(thread.laterFinished, undefined);
1860
+ expect(yield* taskResult(a, followUpId)).toMatchObject({
1861
+ status: "completed",
1862
+ result: "The slow suite passes too.",
1863
+ });
1864
+ }),
1865
+ ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1866
+);
1867
+
1868
+it.effect.each([
1869
+ {
1870
+ restarted: "follow-up",
1871
+ of: boxLaterRunId,
1872
+ laterReply: "Running the slow suite…",
1873
+ restartReply: "The slow suite passes too.",
1874
+ queuedReply: "The flaky suite passes.",
1875
+ },
1876
+ {
1877
+ restarted: "queued",
1878
+ of: boxQueuedRunId,
1879
+ laterReply: "The slow suite passes too.",
1880
+ restartReply: "The flaky suite passes.",
1881
+ queuedReply: "Running the flaky suite…",
1882
+ },
1883
+])(
1884
+ "a follow-up there ends with its own reply when the $restarted run is restarted",
1885
+ ({ of, laterReply, restartReply, queuedReply }) =>
1886
+ Effect.scoped(
1887
+ Effect.gen(function* () {
1888
+ const thread = yield* makeBoxThread;
1889
+ const b = yield* serveBox(thread);
1890
+ const offers = yield* SubscriptionRef.make<
1891
+ ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1892
+ >([]);
1893
+ const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1894
+ const a = yield* makeLaptop(database, offers);
1895
+ yield* a.remote.start();
1896
+ yield* linkTo(a.links, b, "full-access");
1897
+ yield* seedParent(a);
1898
+ const delegated = yield* delegateToBox(a);
1899
+ yield* Ref.set(thread.reply, "All green.");
1900
+ yield* Ref.set(thread.status, "completed");
1901
+ yield* Deferred.succeed(thread.finished, undefined);
1902
+ yield* taskResult(a, delegated.taskId);
1903
+
1904
+ const followUp = yield* a.call("t3_thread_send", {
1905
+ environmentId: box.environmentId,
1906
+ threadId: delegated.remoteChild.threadId,
1907
+ message: "Now run the slow suite too.",
1908
+ });
1909
+ yield* a.call("t3_thread_send", {
1910
+ environmentId: box.environmentId,
1911
+ threadId: delegated.remoteChild.threadId,
1912
+ message: "Then the flaky suite.",
1913
+ mode: "queue",
1914
+ });
1915
+ const followUpId = (followUp.structuredContent as { taskId: NodeId }).taskId;
1916
+
1917
+ // The thread there settles on the queued run, after one of the two was restarted.
1918
+ yield* Ref.set(thread.laterReply, laterReply);
1919
+ yield* Ref.set(thread.queuedReply, queuedReply);
1920
+ yield* Ref.set(thread.restart, { of, reply: restartReply });
1921
+ yield* Deferred.succeed(thread.laterFinished, undefined);
1922
+ expect(yield* taskResult(a, followUpId)).toMatchObject({
1923
+ status: "completed",
1924
+ result: "The slow suite passes too.",
1925
+ });
1926
+ }),
1927
+ ).pipe(Effect.provide(NodeHttpServer.layerTest)),
1928
+);
1929
+
1930
+it.effect("a follow-up sent after the parent's own run ended is still a task", () =>
1931
+ Effect.scoped(
1932
+ Effect.gen(function* () {
1933
+ const thread = yield* makeBoxThread;
1934
+ const b = yield* serveBox(thread);
1935
+ const offers = yield* SubscriptionRef.make<
1936
+ ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>
1937
+ >([]);
1938
+ const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build);
1939
+ const a = yield* makeLaptop(database, offers);
1940
+ yield* a.remote.start();
1941
+ yield* linkTo(a.links, b, "full-access");
1942
+ yield* seedParent(a);
1943
+ const delegated = yield* delegateToBox(a);
1944
+ yield* Ref.set(thread.reply, "All green.");
1945
+ yield* Ref.set(thread.status, "completed");
1946
+ yield* Deferred.succeed(thread.finished, undefined);
1947
+ yield* taskResult(a, delegated.taskId);
1948
+
1949
+ // The parent's turn ends, while a background agent of its own still holds its credential.
1950
+ const now = yield* DateTime.now;
1951
+ const projection = yield* a.orchestrator.getThreadProjection(parentThreadId);
1952
+ const run = projection.runs.find((candidate) => candidate.id === parentRunId)!;
1953
+ yield* a.sink.write({
1954
+ commandId: CommandId.make("command:seed:laptop-parent-ended"),
1955
+ events: [
1956
+ {
1957
+ id: EventId.make("event:seed-background-agent:laptop-parent"),
1958
+ type: "provider-thread.updated",
1959
+ threadId: parentThreadId,
1960
+ driver,
1961
+ providerInstanceId: instanceId,
1962
+ occurredAt: now,
1963
+ payload: {
1964
+ ...projection.providerThreads[0]!,
1965
+ pendingBackgroundTasks: [{ taskId: "native-agent", kind: "subagent" }],
1966
+ updatedAt: now,
1967
+ },
1968
+ },
1969
+ {
1970
+ id: EventId.make("event:seed-run-ended:laptop-parent"),
1971
+ type: "run.updated",
1972
+ threadId: parentThreadId,
1973
+ runId: parentRunId,
1974
+ nodeId: parentRootNode,
1975
+ providerInstanceId: instanceId,
1976
+ occurredAt: now,
1977
+ payload: { ...run, status: "completed", completedAt: now },
1978
+ },
1979
+ ],
1980
+ });
1981
+
1982
+ const sent = yield* a.call("t3_thread_send", {
1983
+ environmentId: box.environmentId,
1984
+ threadId: delegated.remoteChild.threadId,
1985
+ message: "Now run the slow suite too.",
1986
+ });
1987
+ expect(sent.isError, JSON.stringify(sent.content)).toBe(false);
1988
+ const followUp = sent.structuredContent as { taskId?: NodeId };
1989
+ expect(followUp.taskId, JSON.stringify(followUp)).toBeDefined();
1990
+ const tasks = (yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents;
1991
+ expect(tasks.find((task) => task.id === followUp.taskId)).toMatchObject({
1992
+ runId: parentRunId,
1993
+ status: "running",
1994
+ });
1995
+
1996
+ // Its run there ends with its own reply, and the parent wakes for it.
1997
+ const wakes = (
1998
+ seen: ReadonlyArray<ProviderContinuationRequests.ProviderContinuationRequest>,
1999
+ ) => seen.filter((offer) => offer.threadId === parentThreadId).length;
2000
+ const wokenBefore = wakes(yield* SubscriptionRef.get(offers));
2001
+ yield* Ref.set(thread.laterReply, "The slow suite passes too.");
2002
+ yield* Deferred.succeed(thread.laterFinished, undefined);
2003
+ expect(yield* taskResult(a, followUp.taskId!)).toMatchObject({
2004
+ status: "completed",
2005
+ result: "The slow suite passes too.",
2006
+ });
2007
+ const offered = yield* SubscriptionRef.changes(offers).pipe(
2008
+ Stream.filter((seen) => wakes(seen) > wokenBefore),
2009
+ Stream.runHead,
2010
+ );
2011
+ expect(Option.isSome(offered)).toBe(true);
2012
}),
2013
).pipe(Effect.provide(NodeHttpServer.layerTest)),
2014
);
@@ -1759,11 +2039,12 @@ it.effect("a delayed Stop acknowledgement leaves a newer Stop owed", () =>
2039
});
2040
yield* Deferred.await(thread.stopStarted);
2041
// Work starts after the first Stop landed there, but before its acknowledgement returns.
1762
- yield* a.call("t3_thread_send", {
2042
+ const sent = yield* a.call("t3_thread_send", {
2043
environmentId: box.environmentId,
2044
threadId: delegated.remoteChild.threadId,
2045
message: "Run another check.",
2046
});
2047
+ const followUpId = (sent.structuredContent as { taskId: NodeId }).taskId;
2048
yield* a.threads.stopDelegatedTasks({
2049
threadId: parentThreadId,
2050
commandId: CommandId.make("command:stop:delayed-second"),
@@ -1777,7 +2058,9 @@ it.effect("a delayed Stop acknowledgement leaves a newer Stop owed", () =>
2058
status: "completed",
2059
result: "All green.",
2060
});
1780
- expect(yield* Ref.get(thread.interrupted)).toBe(2);
2061
+ // The follow-up the work became is stopped there too, as its own task.
2062
+ expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" });
2063
+ expect(yield* Ref.get(thread.interrupted)).toBe(3);
2064
expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]);
2065
}),
2066
).pipe(Effect.provide(NodeHttpServer.layerTest)),
apps/server/src/peer/RemoteDelegation.ts
+129
-6
@@ -6,6 +6,8 @@ import {
6
type OrchestratorMcpDelegateTaskInput,
7
type OrchestratorMcpTaskCancelResult,
8
type OrchestratorMcpThreadRun,
9
+ type OrchestratorMcpThreadSendInput,
10
+ type OrchestratorMcpThreadSendResult,
11
type OrchestratorMcpThreadTimelineItem,
12
type OrchestrationV2RemoteTaskChild,
13
type OrchestrationV2Subagent,
@@ -17,6 +19,7 @@ import {
19
} from "@t3tools/contracts";
20
import * as Context from "effect/Context";
21
import * as SecureRandom from "effect/crypto/SecureRandom";
22
+import * as DateTime from "effect/DateTime";
23
import * as Duration from "effect/Duration";
24
import * as Effect from "effect/Effect";
25
import * as Layer from "effect/Layer";
@@ -34,6 +37,7 @@ import {
37
resolveRuntimeMode,
38
taskPrompt,
39
} from "../mcp/delegatedTaskModes.ts";
40
+import { followUpOwnerRun } from "../orchestration-v2/CommandPolicy.ts";
41
import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
42
import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts";
43
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
@@ -79,6 +83,18 @@ export class RemoteDelegation extends Context.Service<
83
};
84
},
85
) => Effect.Effect<{ readonly taskId: NodeId }, OrchestratorMcpFailure>;
86
+ /**
87
+ * A `t3_thread_send` there that started or queued a run on the thread of
88
+ * one of this thread's tasks that has its result: like a follow-up to a
89
+ * finished child here, that run is a new task, whose id the send returns.
90
+ * Any other send is returned as it came.
91
+ */
92
+ readonly followUp: (
93
+ scope: McpInvocationContext.McpInvocationScope,
94
+ environmentId: EnvironmentId,
95
+ request: Pick<OrchestratorMcpThreadSendInput, "message" | "clientRequestId">,
96
+ sent: OrchestratorMcpThreadSendResult,
97
+ ) => Effect.Effect<OrchestratorMcpThreadSendResult, OrchestratorMcpFailure>;
98
/**
99
* Stops the task's thread there, then completes it here as cancelled. A
100
* task that already has its result keeps it; its thread there is still
@@ -359,6 +375,90 @@ const make = Effect.gen(function* () {
375
},
376
);
377
378
+ const followUp: RemoteDelegation["Service"]["followUp"] = Effect.fn("RemoteDelegation.followUp")(
379
+ function* (scope, environmentId, request, sent) {
380
+ // A steer joins a run that is already some task's, or no one's.
381
+ if (scope.thread === undefined || (sent.delivery !== "started" && sent.delivery !== "queued"))
382
+ return sent;
383
+ const parentThreadId = scope.thread.threadId;
384
+ const parent = yield* threads
385
+ .getThreadRecords(parentThreadId, ["runs", "subagents"])
386
+ .pipe(Effect.orElseSucceed(() => undefined));
387
+ const isThere = (task: OrchestrationV2Subagent) =>
388
+ task.remoteChild?.environmentId === environmentId &&
389
+ task.remoteChild.threadId === sent.threadId;
390
+ const finished = parent?.subagents.find((task) => isThere(task) && task.result !== null);
391
+ // Bound as a follow-up here is: a background caller's foreground run may have ended.
392
+ const parentRun = parent === undefined ? null : followUpOwnerRun(parent.runs);
393
+ if (
394
+ finished?.remoteChild === undefined ||
395
+ finished.model === null ||
396
+ parentRun === null ||
397
+ parentRun.rootNodeId === null
398
+ )
399
+ return sent;
400
+ // The message id there is the send's, so a retry that reached the same send records once.
401
+ const commandId = CommandId.make(
402
+ `command:remote-followup:${encodeURIComponent(parentThreadId)}:${encodeURIComponent(environmentId)}:${encodeURIComponent(sent.messageId)}`,
403
+ );
404
+ const recorded = yield* threads
405
+ .dispatch({
406
+ type: "delegated_task.remote.request",
407
+ commandId,
408
+ parentThreadId,
409
+ parentRunId: parentRun.id,
410
+ parentNodeId: parentRun.rootNodeId,
411
+ task: request.message,
412
+ ...(finished.title === null ? {} : { title: finished.title }),
413
+ driver: finished.driver,
414
+ modelSelection: { instanceId: finished.providerInstanceId, model: finished.model },
415
+ remoteChild: {
416
+ environmentId,
417
+ threadId: sent.threadId,
418
+ label: finished.remoteChild.label,
419
+ runId: sent.runId,
420
+ },
421
+ completionWake: "always",
422
+ })
423
+ .pipe(
424
+ Effect.tapError((cause) =>
425
+ Effect.logWarning("remote-delegation.followup-record-failed", {
426
+ parentThreadId,
427
+ cause,
428
+ }),
429
+ ),
430
+ Effect.mapError(
431
+ () =>
432
+ new OrchestratorMcpFailure({
433
+ code: "orchestration_error",
434
+ message:
435
+ request.clientRequestId === undefined
436
+ ? "The message was sent, but its task was not recorded; read the thread for its answer instead of sending it again."
437
+ : "The message was sent, but its task was not recorded; send it again with the same clientRequestId to record it.",
438
+ }),
439
+ ),
440
+ );
441
+ const taskEvent = recorded.storedEvents.find(
442
+ (stored) => stored.event.type === "subagent.updated",
443
+ );
444
+ const taskId =
445
+ taskEvent?.event.type === "subagent.updated"
446
+ ? taskEvent.event.payload.id
447
+ : // A replayed request recorded the task the first time.
448
+ yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe(
449
+ Effect.map(
450
+ ({ subagents }) =>
451
+ subagents.find((task) => isThere(task) && task.remoteChild?.runId === sent.runId)
452
+ ?.id,
453
+ ),
454
+ Effect.orElseSucceed(() => undefined),
455
+ );
456
+ if (taskId === undefined) return sent;
457
+ yield* followOne(parentThreadId, taskId).pipe(Effect.forkIn(followers));
458
+ return { ...sent, taskId };
459
+ },
460
+ );
461
+
462
/**
463
* Completes `task` here with its thread there's final state. Each attempt is its own
464
* command, so a failed one never blocks the next; a report refused because the task already
@@ -563,7 +663,9 @@ const make = Effect.gen(function* () {
663
((task.status === "cancelled" || task.status === "interrupted") &&
664
remote.stoppedThere !== true)
665
) {
566
- const key = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`;
666
+ const count = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`;
667
+ // Named for its run too: a follow-up task shares the thread there and counts its own stops.
668
+ const key = remote.runId === undefined ? count : `${count}:${remote.runId}`;
669
yield* stopThere(scope, remote, key).pipe(
670
// Its thread there now runs above what this side may touch, say
671
// because the parent here was lowered since; no retry changes that.
@@ -632,10 +734,31 @@ const make = Effect.gen(function* () {
734
return yield* settled(parentThreadId, taskId);
735
}
736
if (!isTerminal(waited.status)) return false;
635
- const resultRun = waited.resultRun ?? {
636
- runId: remote.runId ?? waited.runId,
637
- status: waited.status,
638
- };
737
+ // A later run's answer there may be another open follow-up's, or its restart, which no run id
738
+ // here names; concurrent sends may record here out of their order there. Then this task's
739
+ // answer is its own run's latest restart, the one waited on.
740
+ const sharedThread =
741
+ waited.resultRun !== undefined &&
742
+ waited.resultRun.runId !== waited.runId &&
743
+ (yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe(
744
+ Effect.map(({ subagents }) => {
745
+ const recordedAt = subagents.find((task) => task.id === taskId)?.startedAt ?? null;
746
+ return subagents.some(
747
+ (other) =>
748
+ other.id !== taskId &&
749
+ other.remoteChild?.environmentId === remote.environmentId &&
750
+ other.remoteChild.threadId === remote.threadId &&
751
+ // One that settled before this was recorded ran before it there.
752
+ (recordedAt === null ||
753
+ other.completedAt === null ||
754
+ DateTime.isGreaterThan(other.completedAt, recordedAt)),
755
+ );
756
+ }),
757
+ ));
758
+ const resultRun =
759
+ waited.resultRun !== undefined && !sharedThread
760
+ ? waited.resultRun
761
+ : { runId: waited.runId, status: waited.status };
762
const result = yield* lastReply(scope, remote, resultRun.runId, resultRun.status === "failed");
763
yield* complete(
764
parentThreadId,
@@ -793,7 +916,7 @@ const make = Effect.gen(function* () {
916
}).pipe(Effect.ignoreCause({ log: true })),
917
);
918
796
- return RemoteDelegation.of({ delegate, cancel, start });
919
+ return RemoteDelegation.of({ delegate, followUp, cancel, start });
920
});
921
922
const isTerminal = (status: string) =>