@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 1,090 lines 44.6 KB
Raw
1 import {
2 CommandId,
3 type EnvironmentId,
4 NodeId,
5 OrchestratorMcpCapabilitiesInput,
6 OrchestratorMcpCapabilitiesResult,
7 OrchestratorMcpFailure,
8 type OrchestratorMcpDelegateTaskInput,
9 type OrchestratorMcpTaskCancelResult,
10 type OrchestratorMcpThreadRun,
11 type OrchestratorMcpThreadSendInput,
12 type OrchestratorMcpThreadSendResult,
13 type OrchestratorMcpThreadTimelineItem,
14 type OrchestrationV2RemoteTaskChild,
15 type OrchestrationV2Subagent,
16 type ProjectId,
17 type ProviderInstanceId,
18 ProviderDriverKind,
19 type RunId,
20 type ThreadId,
21 } from "@t3tools/contracts";
22 import * as Cause from "effect/Cause";
23 import * as Context from "effect/Context";
24 import * as SecureRandom from "effect/crypto/SecureRandom";
25 import * as DateTime from "effect/DateTime";
26 import * as Duration from "effect/Duration";
27 import * as Effect from "effect/Effect";
28 import * as Layer from "effect/Layer";
29 import * as Option from "effect/Option";
30 import * as Schedule from "effect/Schedule";
31 import * as Scope from "effect/Scope";
32 import * as Stream from "effect/Stream";
33 import * as Tool from "effect/ai/Tool";
34 import { withExecutedModelNote } from "@t3tools/provider-core/server/subagentProjection";
35
36 import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
37 import { ProjectToolkit } from "../mcp/toolkits/project/tools.ts";
38 import {
39 resolveInteractionMode,
40 resolveRuntimeMode,
41 taskPrompt,
42 } from "../mcp/delegatedTaskModes.ts";
43 import { followUpOwnerRun } from "../orchestration-v2/CommandPolicy.ts";
44 import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
45 import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts";
46 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
47 import * as ProjectService from "../project/ProjectService.ts";
48 import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts";
49 import { forkParked } from "../serverActivation.ts";
50 import * as PeerForwarding from "./PeerForwarding.ts";
51 import * as PeerLinks from "./PeerLinks.ts";
52
53 /** How long one follow of a remote task waits on its thread there before checking again. */
54 const FOLLOW_WAIT_MS = 10 * 60 * 1_000;
55 /** A linked environment that stops answering is retried, backing off to this. */
56 const FOLLOW_MAX_BACKOFF = Duration.minutes(5);
57 /** A follower that hits this many defects in a row fails its task instead of retrying. */
58 const FOLLOW_MAX_DEFECTS = 3;
59 /** How a follower retries; tests step it by hand. */
60 export const FollowRetrySchedule = Context.Reference<Schedule.Schedule<unknown>>(
61 "t3/peer/RemoteDelegation/FollowRetrySchedule",
62 {
63 defaultValue: () =>
64 Schedule.exponential(Duration.seconds(2)).pipe(
65 Schedule.modifyDelay(({ duration }) =>
66 Effect.succeed(Duration.min(duration, FOLLOW_MAX_BACKOFF)),
67 ),
68 ),
69 },
70 );
71
72 type ThreadScope = McpInvocationContext.McpThreadInvocationScope;
73
74 /** `orchestrator_capabilities` there. Its toolkit imports this module, so it is described here. */
75 const RemoteCapabilitiesTool = Tool.make("orchestrator_capabilities", {
76 parameters: OrchestratorMcpCapabilitiesInput,
77 success: OrchestratorMcpCapabilitiesResult,
78 failure: OrchestratorMcpFailure,
79 failureMode: "return",
80 });
81
82 /**
83 * `delegate_task` to a linked environment. The task's child is an ordinary
84 * thread there, launched through the link; the parent here records the task
85 * like a local one, without a child thread, and a follower completes it when
86 * the thread there ends, so the parent wakes as it would for a local child.
87 */
88 export class RemoteDelegation extends Context.Service<
89 RemoteDelegation,
90 {
91 readonly delegate: (
92 scope: ThreadScope,
93 input: OrchestratorMcpDelegateTaskInput & {
94 readonly target: NonNullable<OrchestratorMcpDelegateTaskInput["target"]> & {
95 readonly environmentId: EnvironmentId;
96 };
97 },
98 ) => Effect.Effect<{ readonly taskId: NodeId }, OrchestratorMcpFailure>;
99 /**
100 * A `t3_thread_send` there that started or queued a run on the thread of
101 * one of this thread's tasks that has its result: like a follow-up to a
102 * finished child here, that run is a new task, whose id the send returns.
103 * Any other send is returned as it came.
104 */
105 readonly followUp: (
106 scope: McpInvocationContext.McpInvocationScope,
107 environmentId: EnvironmentId,
108 request: Pick<OrchestratorMcpThreadSendInput, "message" | "clientRequestId">,
109 sent: OrchestratorMcpThreadSendResult,
110 ) => Effect.Effect<OrchestratorMcpThreadSendResult, OrchestratorMcpFailure>;
111 /**
112 * Stops the task's thread there, then completes it here as cancelled. A
113 * task that already has its result keeps it; its thread there is still
114 * stopped, as cancelling a local task stops follow-up work on its child.
115 */
116 readonly cancel: (
117 scope: ThreadScope,
118 task: OrchestrationV2Subagent,
119 input: {
120 readonly reason?: string | undefined;
121 readonly clientRequestId?: string | undefined;
122 },
123 ) => Effect.Effect<OrchestratorMcpTaskCancelResult, OrchestratorMcpFailure>;
124 /**
125 * The status of the task's run there, or null when it cannot be read: an
126 * unreachable machine reads as no news rather than failing the read here.
127 */
128 readonly runStatus: (
129 scope: ThreadScope,
130 task: OrchestrationV2Subagent,
131 ) => Effect.Effect<OrchestratorMcpThreadRun["status"] | null>;
132 /**
133 * Follows every open remote task until it ends, including after a
134 * restart, and a finished one again when a Stop here owes it a stop there.
135 */
136 readonly start: () => Effect.Effect<void, never, Scope.Scope>;
137 }
138 >()("t3/peer/RemoteDelegation") {}
139
140 /** The other side refused because the thread there runs above this caller's modes. */
141 const isModeRefusal = (error: OrchestratorMcpFailure) =>
142 error.code === "runtime_mode_escalation_denied" ||
143 error.code === "interaction_mode_escalation_denied";
144
145 /**
146 * Refusals that following again cannot change: the link was revoked, expired
147 * or forgotten, or the thread or run there is gone. Anything else (the other
148 * machine asleep, a dropped route) is worth retrying.
149 */
150 const isFinal = (error: OrchestratorMcpFailure) =>
151 error.code === "capability_denied" ||
152 error.code === "thread_not_found" ||
153 error.code === "run_not_found";
154
155 const make = Effect.gen(function* () {
156 const followRetry = yield* FollowRetrySchedule;
157 const forwarding = yield* PeerForwarding.PeerForwarding;
158 const links = yield* PeerLinks.PeerLinks;
159 const threads = yield* ThreadManagement.ThreadManagementService;
160 const projections = yield* ProjectionStore.ProjectionStoreV2;
161 const projects = yield* ProjectService.ProjectService;
162 const repositoryIdentities = yield* RepositoryIdentityResolver.RepositoryIdentityResolver;
163 const random = yield* SecureRandom.SecureRandom;
164 const hereId = yield* ServerEnvironment.ServerEnvironment.pipe(
165 Effect.flatMap((environment) => environment.getEnvironmentId),
166 );
167 // Followers live as long as this service, so shutting down stops them.
168 const followers = yield* Scope.Scope;
169
170 // One follower runs per open task and holds what it read across a wait of
171 // up to FOLLOW_WAIT_MS, so it reads its own task, never the parent's whole
172 // task list: a parent with thousands of tasks made every restart decode
173 // that list once per open task at the same time and ran out of heap.
174 const readTask = (parentThreadId: ThreadId, taskId: NodeId) =>
175 threads
176 .getThreadRecords(parentThreadId, ["subagents"], { subagentIds: [taskId] })
177 .pipe(Effect.map(({ thread, subagents }) => ({ thread, task: subagents[0] })));
178
179 /** The peer's project with this thread's repository, or the one the caller named. */
180 const resolveRemoteProject = Effect.fn("RemoteDelegation.resolveRemoteProject")(function* (
181 scope: ThreadScope,
182 environmentId: EnvironmentId,
183 parentProjectId: ProjectId,
184 requested: ProjectId | undefined,
185 ) {
186 if (requested !== undefined) return requested;
187 const here = yield* projects.getById(parentProjectId).pipe(
188 Effect.map(Option.getOrUndefined),
189 Effect.orElseSucceed(() => undefined),
190 );
191 // The project's own identity is a one-minute cache that reads blank when
192 // cold, so resolve it here, as the other side's t3_project_list does.
193 const key =
194 here === undefined
195 ? undefined
196 : (yield* repositoryIdentities.resolve(here.workspaceRoot))?.canonicalKey;
197 if (key === undefined) {
198 return yield* new OrchestratorMcpFailure({
199 code: "target_required",
200 message: "Pass target.projectId: this thread's project has no repository to match there.",
201 });
202 }
203 const matches: Array<{ readonly id: ProjectId; readonly title: string }> = [];
204 let cursor: number | null = 0;
205 while (cursor !== null) {
206 const listed: Tool.Success<typeof ProjectToolkit.tools.t3_project_list> =
207 yield* forwarding.call(scope, ProjectToolkit.tools.t3_project_list, environmentId, {
208 cursor,
209 limit: 100,
210 });
211 matches.push(
212 ...listed.projects.filter((project) => project.repositoryIdentity?.canonicalKey === key),
213 );
214 cursor = listed.nextCursor;
215 }
216 if (matches.length === 1) return matches[0]!.id;
217 return yield* new OrchestratorMcpFailure({
218 code: "invalid_request",
219 message:
220 matches.length === 0
221 ? `No project there has this repository (${key}). Pass target.projectId; t3_project_list with this environmentId lists them. To bring the repository there, use t3_project_clone and then t3_project_create with this environmentId.`
222 : `Several projects there have this repository: ${matches.map((project) => `${project.title} (${project.id})`).join(", ")}. Pass target.projectId.`,
223 });
224 });
225
226 /** The instance there of the driver the caller named, preferring the one this thread runs on, as a task here does. */
227 const resolveRemoteInstance = Effect.fn("RemoteDelegation.resolveRemoteInstance")(function* (
228 scope: ThreadScope,
229 environmentId: EnvironmentId,
230 requested: {
231 readonly driverKind: ProviderDriverKind;
232 readonly providerInstanceId: ProviderInstanceId | undefined;
233 readonly inheritedInstanceId: ProviderInstanceId;
234 },
235 ) {
236 const { providers } = yield* forwarding.call(scope, RemoteCapabilitiesTool, environmentId, {});
237 if (requested.providerInstanceId !== undefined) {
238 const named = providers.find(
239 (provider) => provider.providerInstanceId === requested.providerInstanceId,
240 );
241 if (named !== undefined && named.driverKind !== requested.driverKind) {
242 return yield* new OrchestratorMcpFailure({
243 code: "invalid_request",
244 message: `Provider instance ${named.providerInstanceId} there uses driver ${named.driverKind}, not ${requested.driverKind}.`,
245 });
246 }
247 return requested.providerInstanceId;
248 }
249 const candidates = providers.filter(
250 (provider) => provider.driverKind === requested.driverKind && provider.canRunChildTask,
251 );
252 const chosen =
253 candidates.find(
254 (provider) => provider.providerInstanceId === requested.inheritedInstanceId,
255 ) ?? candidates[0];
256 if (chosen === undefined) {
257 return yield* new OrchestratorMcpFailure({
258 code: "provider_unavailable",
259 message: `No instance of driver ${requested.driverKind} there can run a child task. orchestrator_capabilities with this environmentId lists its providers.`,
260 });
261 }
262 return chosen.providerInstanceId;
263 });
264
265 const delegate: RemoteDelegation["Service"]["delegate"] = Effect.fn("RemoteDelegation.delegate")(
266 function* (scope, input) {
267 const parent = yield* threads.getThreadRecords(scope.thread.threadId, ["runs"]).pipe(
268 Effect.mapError(
269 () =>
270 new OrchestratorMcpFailure({
271 code: "thread_not_found",
272 message: "The calling thread was not found.",
273 }),
274 ),
275 );
276 const parentRun = parent.runs
277 .filter(ThreadManagement.isActiveRun)
278 .toSorted((left, right) => right.ordinal - left.ordinal)[0];
279 if (
280 parentRun === undefined ||
281 parentRun.rootNodeId === null ||
282 parentRun.providerInstanceId !== scope.thread.providerInstanceId
283 ) {
284 return yield* new OrchestratorMcpFailure({
285 code: "parent_not_active",
286 message: "Delegated tasks require an active run owned by this MCP provider session.",
287 });
288 }
289 const parentNodeId = parentRun.rootNodeId;
290 const { target } = input;
291 // A worktree there would branch from a commit this checkout has and the
292 // other may not, so a remote task runs in that project's checkout.
293 if (input.isolation === "worktree") {
294 return yield* new OrchestratorMcpFailure({
295 code: "invalid_request",
296 message:
297 'A task in a linked environment runs in that project\'s checkout; isolation "worktree" is not supported there. Omit isolation, or use t3_thread_launch with environmentId and a worktree workspaceStrategy.',
298 });
299 }
300 // The provider and model inherit as they do for a task here. Another
301 // provider there has no model to inherit, so the caller names one.
302 const inherited = parent.thread.modelSelection;
303 const instanceId =
304 target.driverKind === undefined
305 ? (target.providerInstanceId ?? inherited.instanceId)
306 : yield* resolveRemoteInstance(scope, target.environmentId, {
307 driverKind: target.driverKind,
308 providerInstanceId: target.providerInstanceId,
309 inheritedInstanceId: inherited.instanceId,
310 });
311 const model =
312 target.model ?? (instanceId === inherited.instanceId ? inherited.model : undefined);
313 if (model === undefined) {
314 return yield* new OrchestratorMcpFailure({
315 code: "target_required",
316 message: "Pass target.model from orchestrator_capabilities with this environmentId.",
317 });
318 }
319 const options =
320 target.options ??
321 (instanceId === inherited.instanceId && model === inherited.model
322 ? inherited.options
323 : undefined);
324 // Modes asked for are checked against this thread. Omitted ones are left
325 // to the other side, which inherits this thread's modes capped by the
326 // link's access, so a narrower link still takes the task.
327 const runtimeMode =
328 input.runtimeMode === undefined || input.runtimeMode === "inherit"
329 ? undefined
330 : yield* resolveRuntimeMode(parent.thread.runtimeMode, input.runtimeMode);
331 const interactionMode =
332 input.interactionMode === undefined || input.interactionMode === "inherit"
333 ? undefined
334 : yield* resolveInteractionMode(parent.thread.interactionMode, input.interactionMode);
335 const projectId = yield* resolveRemoteProject(
336 scope,
337 target.environmentId,
338 parent.thread.projectId,
339 target.projectId,
340 );
341 // One key for the launch there and the record here, so a retry finds both.
342 // Both are scoped to the provider session, as every key a caller passes is:
343 // the forwarder scopes the launch's, and the record's names it.
344 const key = input.clientRequestId ?? (yield* random.uuidV4());
345 const task = taskPrompt(input);
346 const title = input.title ?? parent.thread.title;
347 const launched = yield* forwarding.call(
348 scope,
349 ProjectToolkit.tools.t3_thread_launch,
350 target.environmentId,
351 {
352 projectId,
353 title,
354 modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) },
355 ...(runtimeMode === undefined ? {} : { runtimeMode }),
356 ...(interactionMode === undefined ? {} : { interactionMode }),
357 message: task,
358 clientRequestId: `delegate:${scope.thread.threadId}:${key}`,
359 // The thread there shows it is this thread's subagent, with a way back.
360 delegatedFrom: {
361 environmentId: hereId,
362 threadId: parent.thread.id,
363 title: parent.thread.title,
364 },
365 },
366 );
367 const link = yield* links.get(target.environmentId).pipe(
368 Effect.map(Option.getOrUndefined),
369 Effect.orElseSucceed(() => undefined),
370 );
371 const commandId = CommandId.make(
372 `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`,
373 );
374 const remoteChild: OrchestrationV2RemoteTaskChild = {
375 environmentId: target.environmentId,
376 threadId: launched.threadId,
377 label: link?.label ?? target.environmentId,
378 title,
379 ...(launched.runId === null ? {} : { runId: launched.runId }),
380 };
381 const recorded = yield* threads
382 .dispatch({
383 type: "delegated_task.remote.request",
384 commandId,
385 parentThreadId: scope.thread.threadId,
386 parentRunId: parentRun.id,
387 parentNodeId,
388 task,
389 ...(input.title === undefined ? {} : { title: input.title }),
390 // The driver runs there; this side only labels the task with it.
391 driver: ProviderDriverKind.make("remote"),
392 modelSelection: launched.modelSelection,
393 remoteChild,
394 completionWake: input.mode === "wait" ? "settled_only" : "always",
395 })
396 .pipe(
397 // The cause goes to the log; the caller learns only that recording failed.
398 Effect.tapError((cause) =>
399 Effect.logWarning("remote-delegation.record-failed", {
400 parentThreadId: scope.thread.threadId,
401 cause,
402 }),
403 ),
404 Effect.mapError(
405 () =>
406 new OrchestratorMcpFailure({
407 code: "orchestration_error",
408 message: "Unable to record the delegated task.",
409 }),
410 ),
411 // Nothing here follows a thread the task was never recorded for, so stop it there.
412 // A crash between the launch and this record still leaves it running there
413 // until a retry with the same clientRequestId records it.
414 Effect.tapError(() =>
415 stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe(
416 Effect.ignoreCause({ log: true }),
417 ),
418 ),
419 );
420 const taskEvent = recorded.storedEvents.find(
421 (stored) => stored.event.type === "subagent.updated",
422 );
423 const taskId =
424 taskEvent?.event.type === "subagent.updated"
425 ? taskEvent.event.payload.id
426 : // A replayed request recorded the task the first time.
427 yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe(
428 Effect.map(
429 ({ subagents }) =>
430 subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id,
431 ),
432 Effect.orElseSucceed(() => undefined),
433 );
434 if (taskId === undefined) {
435 return yield* new OrchestratorMcpFailure({
436 code: "orchestration_error",
437 message: "The delegated task was not recorded.",
438 });
439 }
440 yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers));
441 return { taskId };
442 },
443 );
444
445 const followUp: RemoteDelegation["Service"]["followUp"] = Effect.fn("RemoteDelegation.followUp")(
446 function* (scope, environmentId, request, sent) {
447 // A steer joins a run that is already some task's, or no one's.
448 if (scope.thread === undefined || (sent.delivery !== "started" && sent.delivery !== "queued"))
449 return sent;
450 const parentThreadId = scope.thread.threadId;
451 const parent = yield* threads
452 .getThreadRecords(parentThreadId, ["runs", "subagents"])
453 .pipe(Effect.orElseSucceed(() => undefined));
454 const isThere = (task: OrchestrationV2Subagent) =>
455 task.remoteChild?.environmentId === environmentId &&
456 task.remoteChild.threadId === sent.threadId;
457 const finished = parent?.subagents.find((task) => isThere(task) && task.result !== null);
458 // Bound as a follow-up here is: a background caller's foreground run may have ended.
459 const parentRun = parent === undefined ? null : followUpOwnerRun(parent.runs);
460 if (
461 finished?.remoteChild === undefined ||
462 finished.model === null ||
463 parentRun === null ||
464 parentRun.rootNodeId === null
465 )
466 return sent;
467 // The message id there is the send's, so a retry that reached the same send records once.
468 const commandId = CommandId.make(
469 `command:remote-followup:${encodeURIComponent(parentThreadId)}:${encodeURIComponent(environmentId)}:${encodeURIComponent(sent.messageId)}`,
470 );
471 const recorded = yield* threads
472 .dispatch({
473 type: "delegated_task.remote.request",
474 commandId,
475 parentThreadId,
476 parentRunId: parentRun.id,
477 parentNodeId: parentRun.rootNodeId,
478 task: request.message,
479 ...(finished.title === null ? {} : { title: finished.title }),
480 driver: finished.driver,
481 modelSelection: { instanceId: finished.providerInstanceId, model: finished.model },
482 remoteChild: {
483 environmentId,
484 threadId: sent.threadId,
485 label: finished.remoteChild.label,
486 runId: sent.runId,
487 },
488 completionWake: "always",
489 })
490 .pipe(
491 Effect.tapError((cause) =>
492 Effect.logWarning("remote-delegation.followup-record-failed", {
493 parentThreadId,
494 cause,
495 }),
496 ),
497 Effect.mapError(
498 () =>
499 new OrchestratorMcpFailure({
500 code: "orchestration_error",
501 message:
502 request.clientRequestId === undefined
503 ? "The message was sent, but its task was not recorded; read the thread for its answer instead of sending it again."
504 : "The message was sent, but its task was not recorded; send it again with the same clientRequestId to record it.",
505 }),
506 ),
507 );
508 const taskEvent = recorded.storedEvents.find(
509 (stored) => stored.event.type === "subagent.updated",
510 );
511 const taskId =
512 taskEvent?.event.type === "subagent.updated"
513 ? taskEvent.event.payload.id
514 : // A replayed request recorded the task the first time.
515 yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe(
516 Effect.map(
517 ({ subagents }) =>
518 subagents.find((task) => isThere(task) && task.remoteChild?.runId === sent.runId)
519 ?.id,
520 ),
521 Effect.orElseSucceed(() => undefined),
522 );
523 if (taskId === undefined) return sent;
524 yield* followOne(parentThreadId, taskId).pipe(Effect.forkIn(followers));
525 return { ...sent, taskId };
526 },
527 );
528
529 /**
530 * Completes `task` here with its thread there's final state. Each attempt is its own
531 * command, so a failed one never blocks the next; a report refused because the task already
532 * has its result (a cancel, a Stop or an earlier report) is done. `stoppedThere` records that
533 * nothing is left to stop there, even on a task that already has its result.
534 */
535 const complete = Effect.fn("RemoteDelegation.complete")(function* (
536 parentThreadId: ThreadId,
537 taskId: NodeId,
538 status: "completed" | "failed" | "cancelled" | "interrupted",
539 result: string,
540 stoppedThere: "ran-out" | "stopped" | undefined,
541 stopCount?: number,
542 executedModel?: string,
543 ) {
544 const attempt = yield* random.uuidV4();
545 yield* threads
546 .dispatch({
547 type: "delegated_task.remote.complete",
548 commandId: CommandId.make(`command:remote-task-complete:${taskId}:${attempt}`),
549 parentThreadId,
550 taskId,
551 status,
552 result,
553 ...(stoppedThere === undefined ? {} : { stoppedThere }),
554 ...(stopCount === undefined ? {} : { stopCount }),
555 ...(executedModel === undefined ? {} : { executedModel }),
556 })
557 .pipe(
558 Effect.catch((error) =>
559 readTask(parentThreadId, taskId).pipe(
560 Effect.flatMap(({ task }) => (task?.result != null ? Effect.void : Effect.fail(error))),
561 ),
562 ),
563 );
564 });
565
566 /**
567 * How this environment reaches a task's thread there: as the parent
568 * thread, under one namespace per parent, so a stop sent again after a
569 * restart or a dropped route is recorded there once.
570 */
571 const followScope = (
572 parent: { readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId },
573 remote: OrchestrationV2RemoteTaskChild,
574 ): ThreadScope => ({
575 environmentId: hereId,
576 requestNamespace: `remote-task:${parent.threadId}`,
577 thread: {
578 threadId: parent.threadId,
579 providerSessionId: `remote-task:${remote.threadId}`,
580 providerInstanceId: parent.providerInstanceId,
581 },
582 client: undefined,
583 capabilities: new Set(["orchestration"]),
584 issuedAt: 0,
585 });
586
587 /**
588 * Stops the task's own thread there as its Stop button does: its running turn, queued
589 * messages, pull request watches and delegated tasks. `key` names the stop (the follower
590 * sends one, each cancel its own), and each is recorded there once however often it is sent.
591 */
592 const stopThere = (
593 scope: ThreadScope,
594 remote: OrchestrationV2RemoteTaskChild,
595 key: string,
596 reason?: string,
597 ) =>
598 forwarding
599 .interruptThread(scope, remote.environmentId, {
600 threadId: remote.threadId,
601 stop: true,
602 ...(reason === undefined ? {} : { reason }),
603 clientRequestId: `${key}:${remote.threadId}`,
604 })
605 .pipe(Effect.asVoid);
606
607 /**
608 * The last reply `runId` gave on the task's thread there, which is its
609 * result, read whole however long it is. A failed run prefers its error
610 * from the activity view over an earlier reply, without taking another run's.
611 * Led by a note when its provider reported running another model than it asked for.
612 */
613 const lastReply = Effect.fn("RemoteDelegation.lastReply")(function* (
614 scope: ThreadScope,
615 remote: OrchestrationV2RemoteTaskChild,
616 runId: RunId | null,
617 failed = false,
618 probeSuccess = true,
619 ) {
620 let reply: OrchestratorMcpThreadTimelineItem | undefined;
621 let error: OrchestratorMcpThreadTimelineItem | undefined;
622 let afterPosition: number | undefined;
623 let run: OrchestratorMcpThreadRun | undefined;
624 let remainingItems = 0;
625 let tailProbed = false;
626 while (true) {
627 const read = yield* forwarding.readThread(scope, remote.environmentId, {
628 threadId: remote.threadId,
629 view: failed ? "activity" : "messages",
630 limit: 100,
631 runLimit: 50,
632 // Keep the first and expected final activity pages whole. A result on a
633 // middle page then needs an extra text read after the scan.
634 maxCharsPerItem: failed && remainingItems > 100 ? 1 : 20_000,
635 ...(afterPosition === undefined ? {} : { afterPosition }),
636 });
637 run ??= read.recentRuns.find((candidate) => candidate.runId === runId);
638 reply =
639 read.items.findLast(
640 (item) => item.type === "assistant_message" && (runId === null || item.runId === runId),
641 ) ?? reply;
642 if (failed) {
643 error =
644 read.items.findLast(
645 (item) => item.type === "error" && (runId === null || item.runId === runId),
646 ) ?? error;
647 }
648 if (!read.hasMore || read.nextPosition === null) break;
649 if (!failed && probeSuccess && !tailProbed) {
650 tailProbed = true;
651 const tailPosition = Math.floor((read.thread.itemCount - 1) / 100) * 100 - 1;
652 if (tailPosition > read.nextPosition) {
653 const tail = yield* forwarding.readThread(scope, remote.environmentId, {
654 threadId: remote.threadId,
655 view: "messages",
656 limit: 100,
657 maxCharsPerItem: 20_000,
658 afterPosition: tailPosition,
659 });
660 const tailReply = tail.items.findLast(
661 (item) => item.type === "assistant_message" && (runId === null || item.runId === runId),
662 );
663 if (!tail.hasMore && tailReply !== undefined) {
664 reply = tailReply;
665 break;
666 }
667 }
668 }
669 afterPosition = read.nextPosition;
670 remainingItems = read.thread.itemCount - afterPosition - 1;
671 }
672 reply = error ?? reply;
673 const executedModel = run?.executedModel ?? undefined;
674 if (reply === undefined) return { text: undefined, executedModel };
675 let text = reply.text ?? "";
676 let textOffset = reply.textTruncated ? (reply.nextTextOffset ?? null) : null;
677 while (textOffset !== null) {
678 const rest = yield* forwarding.readThread(scope, remote.environmentId, {
679 threadId: remote.threadId,
680 itemId: reply.itemId,
681 textOffset,
682 maxCharsPerItem: 20_000,
683 });
684 const part = rest.items[0];
685 if (part === undefined) break;
686 text += part.text ?? "";
687 textOffset = part.textTruncated ? (part.nextTextOffset ?? null) : null;
688 }
689 return {
690 text: run === undefined ? text : withExecutedModelNote(text, run.model, run.executedModel),
691 executedModel,
692 };
693 });
694
695 /**
696 * Why following a task can never succeed, if that is what `error` means.
697 * The far side's message already says what was refused and what to do; a
698 * link forgotten here only reads as a bad request, so it is named.
699 */
700 const finalReason = Effect.fn("RemoteDelegation.finalReason")(
701 function* (parentThreadId: ThreadId, taskId: NodeId, error: OrchestratorMcpFailure) {
702 if (isFinal(error)) return error.message;
703 if (error.code !== "invalid_request") return undefined;
704 const { task } = yield* readTask(parentThreadId, taskId);
705 const environmentId = task?.remoteChild?.environmentId;
706 if (environmentId === undefined) return error.message;
707 const link = yield* links.get(environmentId);
708 return Option.isSome(link) ? undefined : error.message;
709 },
710 Effect.orElseSucceed(() => undefined),
711 );
712
713 /** A task's thread there that ended without a reply. */
714 const noReply = "(The thread in the linked environment ended without a reply.)";
715
716 /**
717 * Waits for the task's run there to end, completes the task here, and stops it there if it
718 * ended here first. A revoked, expired or forgotten link, or a thread or run gone there,
719 * fails the task with nothing left to stop; repeated defects here fail it too. Anything else
720 * (the other machine asleep, a dropped route) is retried with back-off, and so is completing.
721 */
722 const follow = (parentThreadId: ThreadId, taskId: NodeId): Effect.Effect<void> =>
723 Effect.suspend(() => {
724 let defects = 0;
725 const noDefect = Effect.sync(() => {
726 defects = 0;
727 });
728 return followOnce(parentThreadId, taskId).pipe(
729 Effect.tap(noDefect),
730 // `false` means it is still running there: follow again straight away.
731 Effect.repeat({ while: (done) => !done }),
732 Effect.catchTags({
733 OrchestratorMcpFailure: Effect.fn("RemoteDelegation.followFailed")(function* (error) {
734 const reason = yield* finalReason(parentThreadId, taskId, error);
735 if (reason === undefined) return yield* error;
736 // Gone there, or no link to reach it by: nothing left to stop there.
737 const { task } = yield* readTask(parentThreadId, taskId);
738 const stopCount = task?.remoteChild?.stopCount;
739 yield* complete(parentThreadId, taskId, "failed", reason, "stopped", stopCount);
740 }),
741 }),
742 Effect.tapError(() => noDefect),
743 Effect.tapDefect(() =>
744 Effect.sync(() => {
745 defects += 1;
746 }),
747 ),
748 failOnDefect,
749 Effect.retry({ schedule: followRetry, while: () => defects < FOLLOW_MAX_DEFECTS }),
750 Effect.catch(
751 Effect.fn("RemoteDelegation.followGaveUp")(
752 function* () {
753 const { task } = yield* readTask(parentThreadId, taskId);
754 const remote = task?.remoteChild;
755 // A task with its result keeps it; completing it again would owe another stop there.
756 if (remote === undefined || task?.result !== null) return;
757 const result = `Following the thread ${remote.threadId} in ${remote.label} failed repeatedly, so it may still run there (task_cancel stops it).`;
758 yield* complete(parentThreadId, taskId, "failed", result, undefined);
759 },
760 failOnDefect,
761 Effect.retry(followRetry),
762 ),
763 ),
764 Effect.ignoreCause({ log: true }),
765 );
766 });
767
768 /**
769 * One wait on the task's thread there; true once the task here is done.
770 * A task completed here first (cancelled, or its parent stopped) is
771 * stopped there, and done once that stop is recorded here.
772 */
773 const followOnce = Effect.fn("RemoteDelegation.followOnce")(function* (
774 parentThreadId: ThreadId,
775 taskId: NodeId,
776 ) {
777 const sequence = yield* threads.getThreadEventSequence(parentThreadId);
778 const { thread, task } = yield* readTask(parentThreadId, taskId);
779 if (task?.remoteChild === undefined) return true;
780 const remote = task.remoteChild;
781 const scope = followScope(
782 { threadId: parentThreadId, providerInstanceId: thread.providerInstanceId },
783 remote,
784 );
785 // Ended here, maybe while this follower backed off from an unreachable
786 // machine or before a restart: it still has to be stopped there. So
787 // does one its parent here was stopped after, for what followed there.
788 if (task.result !== null) {
789 if (
790 remote.stoppedThere === false ||
791 ((task.status === "cancelled" || task.status === "interrupted") &&
792 remote.stoppedThere !== true)
793 ) {
794 const count = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`;
795 // Named for its run too: a follow-up task shares the thread there and counts its own stops.
796 const key = remote.runId === undefined ? count : `${count}:${remote.runId}`;
797 yield* stopThere(scope, remote, key).pipe(
798 // Its thread there now runs above what this side may touch, say
799 // because the parent here was lowered since; no retry changes that.
800 Effect.catchIf(isModeRefusal, (error) =>
801 Effect.logWarning("A linked environment refused to stop a delegated task", {
802 taskId,
803 reason: error.message,
804 }),
805 ),
806 );
807 yield* complete(
808 parentThreadId,
809 taskId,
810 terminalStatus(task.status),
811 task.result,
812 "stopped",
813 remote.stopCount,
814 );
815 return yield* settled(parentThreadId, taskId);
816 }
817 return true;
818 }
819 const endedHere = threads
820 .streamStoredEventsFrom({
821 threadId: parentThreadId,
822 afterSequence: sequence,
823 eventType: "subagent.updated",
824 })
825 .pipe(
826 Stream.filter(
827 (stored) =>
828 stored.event.type === "subagent.updated" &&
829 stored.event.payload.id === taskId &&
830 stored.event.payload.result !== null,
831 ),
832 Stream.runHead,
833 Effect.flatMap((ended) =>
834 Option.isSome(ended) ? Effect.succeed({ endedHere: true } as const) : Effect.never,
835 ),
836 );
837 const waited = yield* forwarding
838 .waitForThread(scope, remote.environmentId, {
839 threadId: remote.threadId,
840 ...(remote.runId === undefined ? {} : { runId: remote.runId }),
841 timeoutMs: FOLLOW_WAIT_MS,
842 // A thread there whose run ended may still be waiting on its own
843 // tasks to wake it for the answer, which a task here waits for too.
844 // An older peer ignores this and reports the run alone.
845 untilDelegatedWorkSettled: true,
846 })
847 .pipe(Effect.raceFirst(endedHere));
848 // The next look reads the result and stops it there.
849 if ("endedHere" in waited) return false;
850 if (waited.timedOut) return false;
851 // A task recorded without its run waits on the thread's latest; a
852 // thread there with none has nothing left to wait for.
853 if (waited.runId === null) {
854 const reply = yield* lastReply(scope, remote, null);
855 yield* complete(
856 parentThreadId,
857 taskId,
858 reply.text === undefined ? "failed" : "completed",
859 reply.text ?? "The thread in the linked environment has no run to follow.",
860 "ran-out",
861 );
862 return yield* settled(parentThreadId, taskId);
863 }
864 if (!isTerminal(waited.status)) return false;
865 // A later run's answer there may be another open follow-up's, or its restart, which no run id
866 // here names; concurrent sends may record here out of their order there. Then this task's
867 // answer is its own run's latest restart, the one waited on.
868 const sharedThread =
869 waited.resultRun !== undefined &&
870 waited.resultRun.runId !== waited.runId &&
871 (yield* threads.getThreadRecords(parentThreadId, ["subagents"]).pipe(
872 Effect.map(({ subagents }) => {
873 const recordedAt = subagents.find((task) => task.id === taskId)?.startedAt ?? null;
874 return subagents.some(
875 (other) =>
876 other.id !== taskId &&
877 other.remoteChild?.environmentId === remote.environmentId &&
878 other.remoteChild.threadId === remote.threadId &&
879 // One that settled before this was recorded ran before it there.
880 (recordedAt === null ||
881 other.completedAt === null ||
882 DateTime.isGreaterThan(other.completedAt, recordedAt)),
883 );
884 }),
885 ));
886 const resultRun =
887 waited.resultRun !== undefined && !sharedThread
888 ? waited.resultRun
889 : { runId: waited.runId, status: waited.status };
890 const result = yield* lastReply(
891 scope,
892 remote,
893 resultRun.runId,
894 resultRun.status === "failed",
895 resultRun.status === "completed",
896 );
897 yield* complete(
898 parentThreadId,
899 taskId,
900 terminalStatus(resultRun.status),
901 result.text ?? noReply,
902 "ran-out",
903 undefined,
904 result.executedModel,
905 );
906 return yield* settled(parentThreadId, taskId);
907 });
908
909 /**
910 * Whether the task is done here after the follower reported its run ran
911 * out there. A cancel or Stop that landed while the reply was read keeps
912 * its result, and still owes a stop there for what may follow the run;
913 * the next look sends it.
914 */
915 const settled = (parentThreadId: ThreadId, taskId: NodeId) =>
916 readTask(parentThreadId, taskId).pipe(
917 Effect.map(({ task }) => task?.remoteChild?.stoppedThere === true),
918 );
919
920 const following = new Map<NodeId, { again: boolean }>();
921
922 /**
923 * `follow`, at most once per task. Asked again while it runs, say by a
924 * Stop here on a task that already has its result, it looks once more
925 * before it ends, so that stop is not missed.
926 */
927 const followOne = (parentThreadId: ThreadId, taskId: NodeId) =>
928 Effect.suspend(() => {
929 const running = following.get(taskId);
930 if (running !== undefined) {
931 running.again = true;
932 return Effect.void;
933 }
934 const state = { again: false };
935 following.set(taskId, state);
936 return follow(parentThreadId, taskId).pipe(
937 // Checked and released in one step, so a request between them is never lost.
938 Effect.repeat({
939 while: () => {
940 if (state.again) {
941 state.again = false;
942 return true;
943 }
944 following.delete(taskId);
945 return false;
946 },
947 }),
948 Effect.ensuring(
949 Effect.sync(() => {
950 if (following.get(taskId) === state) following.delete(taskId);
951 }),
952 ),
953 );
954 });
955
956 const cancel: RemoteDelegation["Service"]["cancel"] = Effect.fn("RemoteDelegation.cancel")(
957 function* (scope, task, input) {
958 const remote = task.remoteChild;
959 if (remote === undefined) {
960 return yield* new OrchestratorMcpFailure({
961 code: "task_not_found",
962 message: `Delegated task ${task.id} is not remote.`,
963 });
964 }
965 const { reason } = input;
966 const there = followScope(scope.thread, remote);
967 // A later cancel stops what started there since; a retry of this one, nothing new.
968 const key = `cancel:${encodeURIComponent(scope.requestNamespace)}:${
969 input.clientRequestId ?? (yield* random.uuidV4())
970 }`;
971 if (task.result !== null) {
972 // Best effort: the result stands whether or not the stop gets there.
973 yield* stopThere(there, remote, key, reason).pipe(
974 Effect.andThen(
975 complete(
976 task.threadId,
977 task.id,
978 terminalStatus(task.status),
979 task.result,
980 "stopped",
981 remote.stopCount,
982 ),
983 ),
984 Effect.ignoreCause({ log: true }),
985 );
986 return {
987 taskId: task.id,
988 status:
989 task.status === "completed" || task.status === "failed" || task.status === "interrupted"
990 ? task.status
991 : "cancelled",
992 } satisfies OrchestratorMcpTaskCancelResult;
993 }
994 // A link that is gone or a machine that is down still cancels the task
995 // here; its follower keeps trying to stop it there, across restarts.
996 const unstopped = yield* stopThere(there, remote, key, reason).pipe(
997 Effect.as(undefined),
998 Effect.catch((error) => Effect.succeed(error.message)),
999 );
1000 const cancelled = reason === undefined ? "Cancelled." : `Cancelled: ${reason}`;
1001 yield* complete(
1002 task.threadId,
1003 task.id,
1004 "cancelled",
1005 unstopped === undefined
1006 ? cancelled
1007 : `${cancelled} It could not be stopped in ${remote.label} and may still run there: ${unstopped}`,
1008 unstopped === undefined ? "stopped" : undefined,
1009 remote.stopCount,
1010 ).pipe(
1011 Effect.tapError((cause) =>
1012 Effect.logWarning("remote-delegation.cancel-record-failed", { taskId: task.id, cause }),
1013 ),
1014 Effect.mapError(
1015 () =>
1016 new OrchestratorMcpFailure({
1017 code: "orchestration_error",
1018 message: "Unable to record the cancellation.",
1019 }),
1020 ),
1021 );
1022 return {
1023 taskId: task.id,
1024 status: "cancel_requested",
1025 } satisfies OrchestratorMcpTaskCancelResult;
1026 },
1027 );
1028
1029 const runStatus = Effect.fn("RemoteDelegation.runStatus")(
1030 function* (scope: ThreadScope, task: OrchestrationV2Subagent) {
1031 const remote = task.remoteChild;
1032 if (remote?.runId === undefined) return null;
1033 const read = yield* forwarding.readThread(
1034 followScope(scope.thread, remote),
1035 remote.environmentId,
1036 { threadId: remote.threadId, limit: 1, runLimit: 50 },
1037 );
1038 return read.recentRuns.find((run) => run.runId === remote.runId)?.status ?? null;
1039 },
1040 Effect.orElseSucceed(() => null),
1041 );
1042
1043 const start: RemoteDelegation["Service"]["start"] = () =>
1044 forkParked(
1045 Effect.gen(function* () {
1046 // A finished task's follower has ended, so a Stop here that owes its
1047 // thread there a stop starts it again. Started before the scan below;
1048 // a Stop that slips between the two stays owed for the next start.
1049 yield* Stream.runForEach(threads.streamDomainEvents, (event) =>
1050 event.type === "subagent.updated" && event.payload.remoteChild?.stoppedThere === false
1051 ? followOne(event.threadId, event.payload.id).pipe(Effect.forkIn(followers))
1052 : Effect.void,
1053 ).pipe(
1054 Effect.ignoreCause({ log: true }),
1055 Effect.forkIn(followers, { startImmediately: true }),
1056 );
1057 const open = yield* projections.getOpenRemoteDelegatedTasks;
1058 yield* Effect.forEach(
1059 open,
1060 ({ parentThreadId, taskId }) => followOne(parentThreadId, taskId),
1061 {
1062 concurrency: "unbounded",
1063 discard: true,
1064 },
1065 );
1066 }).pipe(Effect.ignoreCause({ log: true })),
1067 );
1068
1069 return RemoteDelegation.of({ delegate, followUp, cancel, runStatus, start });
1070 });
1071
1072 /** A defect, logged, as a failure, so a retry covers it. */
1073 const failOnDefect = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
1074 Effect.catchDefect(effect, (defect) =>
1075 Effect.logWarning("A remote task's follower hit a defect", Cause.die(defect)).pipe(
1076 Effect.andThen(Effect.fail(new Cause.UnknownError(defect, "A follower hit a defect."))),
1077 ),
1078 );
1079
1080 const isTerminal = (status: string) =>
1081 status === "completed" ||
1082 status === "failed" ||
1083 status === "cancelled" ||
1084 status === "interrupted" ||
1085 status === "rolled_back";
1086
1087 const terminalStatus = (status: string): "completed" | "failed" | "cancelled" | "interrupted" =>
1088 status === "completed" || status === "failed" || status === "interrupted" ? status : "cancelled";
1089
1090 export const layer = Layer.effect(RemoteDelegation, make);