@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 5,965 lines 232 KB
Raw
1 import * as Exit from "effect/Exit";
2 import * as EffectOutbox from "./EffectOutbox.ts";
3 import * as EffectWorker from "./EffectWorker.ts";
4 import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
5 import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
6 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
7 import * as NodeServices from "@effect/platform-node/NodeServices";
8 import { assert, it, vi } from "@effect/vitest";
9 import {
10 CheckpointId,
11 CommandId,
12 EventId,
13 MessageId,
14 type ModelSelection,
15 NodeId,
16 type OrchestrationV2Run,
17 ProjectId,
18 ProviderDriverKind,
19 ProviderInstanceId,
20 ProviderSessionId,
21 ProviderThreadId,
22 RunId,
23 RuntimeRequestId,
24 ThreadId,
25 TurnItemId,
26 } from "@t3tools/contracts";
27 import * as DateTime from "effect/DateTime";
28 import * as Effect from "effect/Effect";
29 import * as Deferred from "effect/Deferred";
30 import * as Fiber from "effect/Fiber";
31 import * as Layer from "effect/Layer";
32 import * as Stream from "effect/Stream";
33 import * as Tracer from "effect/Tracer";
34 import * as TestClock from "effect/testing/TestClock";
35
36 import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
37 import * as ServerConfig from "../config.ts";
38 import * as McpSessionRegistryTestkit from "../mcp/McpSessionRegistry.testkit.ts";
39 import * as SqlitePersistence from "../persistence/Sqlite.ts";
40 import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts";
41 import * as ProjectService from "../project/ProjectService.ts";
42 import type { ProviderInstance } from "@t3tools/provider-core/server/driver";
43 import * as ModelManifest from "../provider/ModelManifest.ts";
44 import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
45 import * as ServerSettings from "../serverSettings.ts";
46 import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
47 import * as VcsProcess from "../vcs/VcsProcess.ts";
48 import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
49 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
50 import * as EventSink from "./EventSink.ts";
51 import * as Orchestrator from "./Orchestrator.ts";
52 import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
53 import { continueRestartedRun } from "./RestartContinuation.ts";
54 import * as RuntimeLayer from "./runtimeLayer.ts";
55 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
56 import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
57 import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
58 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
59 import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
60 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
61 import { PROVIDER_STOP_REASON_SIGNAL } from "@t3tools/provider-core/server/subagentProjection";
62 import * as ProviderContinuationService from "./ProviderContinuationService.ts";
63
64 const layerPlatformTest = Layer.merge(
65 Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
66 Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({
67 resolveLink: () => Effect.die("unused title link"),
68 }),
69 );
70
71 const layerServerConfig = ServerConfig.layerTest(process.cwd(), {
72 prefix: "t3-orchestration-v2-delegated-completion-",
73 });
74
75 const modelSelection = {
76 instanceId: ProviderInstanceId.make("codex"),
77 model: "gpt-5.4",
78 } satisfies ModelSelection;
79
80 const layerVcsDriverRegistryTest = VcsDriverRegistry.layer.pipe(
81 Layer.provide(VcsProcess.layer),
82 Layer.provide(layerServerConfig),
83 Layer.provide(layerPlatformTest),
84 );
85
86 const layerCheckpointStoreTest = CheckpointStore.layer.pipe(
87 Layer.provide(layerVcsDriverRegistryTest),
88 );
89
90 const driver = ProviderDriverKind.make("codex");
91 const orchestrationAdapter = {
92 instanceId: modelSelection.instanceId,
93 driver,
94 getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
95 planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }),
96 openSession: () => Effect.die("sessions are not used by delegated completion tests"),
97 } as ProviderAdapter.ProviderAdapterV2["Service"];
98 const providerInstance = {
99 instanceId: modelSelection.instanceId,
100 driverKind: driver,
101 continuationIdentity: {
102 driverKind: driver,
103 continuationKey: "codex:test",
104 },
105 displayName: "Codex test",
106 enabled: true,
107 snapshot: { getSnapshot: Effect.succeed({}) } as unknown as ProviderInstance["snapshot"],
108 orchestrationAdapter,
109 textGeneration: {} as ProviderInstance["textGeneration"],
110 } satisfies ProviderInstance;
111
112 const layerTestProviderInstanceRegistry = Layer.merge(
113 ModelManifest.layerTest,
114 Layer.succeed(ProviderInstanceRegistry.ProviderInstanceRegistry, {
115 getInstance: (instanceId) =>
116 Effect.succeed(instanceId === providerInstance.instanceId ? providerInstance : undefined),
117 listInstances: Effect.succeed([providerInstance]),
118 listUnavailable: Effect.succeed([]),
119 streamChanges: Stream.empty,
120 subscribeChanges: Effect.never,
121 }),
122 );
123
124 const layerTest = Layer.mergeAll(
125 RuntimeLayer.layer,
126 RuntimeLayer.layerEventSink,
127 ThreadCommandExecutor.layer,
128 EffectOutbox.layer,
129 ).pipe(
130 Layer.provideMerge(RuntimeLayer.layerProjectService),
131 Layer.provide(
132 Layer.mock(WorkspacePaths.WorkspacePaths)({
133 normalizeWorkspaceRoot: (workspaceRoot) => Effect.succeed(workspaceRoot),
134 }),
135 ),
136 Layer.provide(ProviderTurnStartServiceTestkit.layer),
137 Layer.provide(
138 Layer.succeed(ProjectEnrichmentService.ProjectEnrichmentService, {
139 peek: () =>
140 Effect.succeed({
141 repositoryIdentity: null,
142 faviconPath: null,
143 repositoryIdentityResolved: false,
144 }),
145 request: () => Effect.void,
146 readRepositoryIdentity: () => Effect.succeed(null),
147 getAvailable: () =>
148 Effect.succeed({
149 repositoryIdentity: null,
150 faviconPath: null,
151 repositoryIdentityResolved: false,
152 }),
153 invalidate: () => Effect.void,
154 refreshRepositoryIdentity: () => Effect.void,
155 subscribeChanges: Effect.never,
156 }),
157 ),
158 Layer.provide(McpSessionRegistryTestkit.layer),
159 Layer.provide(McpProviderSessions.layer),
160 Layer.provide(SqlitePersistence.layerMemory),
161 Layer.provide(layerCheckpointStoreTest),
162 Layer.provide(layerServerConfig),
163 Layer.provide(ServerSettings.layerTest()),
164 Layer.provide(layerTestProviderInstanceRegistry),
165 Layer.provide(layerPlatformTest),
166 );
167
168 // ProviderContinuationService consumes the orchestrator's continuation queue, as in production.
169 const layerContinuationWorkerTest = ProviderContinuationService.layer.pipe(
170 Layer.provide(Layer.merge(ProviderContinuationRequests.layer, IdAllocator.layer)),
171 Layer.provideMerge(layerTest),
172 );
173
174 const seedParentWithTerminalTask = (input: {
175 readonly threadId: ThreadId;
176 readonly projectId: ProjectId;
177 readonly runId: RunId;
178 readonly rootNodeId: NodeId;
179 readonly taskId: NodeId;
180 readonly deliveryState: "pending" | "delivered" | "claimed" | "acknowledged" | "disposed";
181 readonly completionWake?: "always" | "settled_only";
182 readonly deliveryTaskIds?: ReadonlyArray<NodeId>;
183 readonly parentRunStatus?: "running" | "completed";
184 readonly siblingTasks?: ReadonlyArray<{
185 readonly id: NodeId;
186 readonly deliveryState: "delivered" | "claimed";
187 }>;
188 readonly now: DateTime.Utc;
189 }) =>
190 Effect.gen(function* () {
191 const projects = yield* ProjectService.ProjectService;
192 const orchestrator = yield* Orchestrator.OrchestratorV2;
193 const eventSink = yield* EventSink.EventSinkV2;
194 const providerThreadId = ProviderThreadId.make(
195 `provider-thread:${String(input.threadId).replace("thread:", "")}`,
196 );
197
198 yield* projects.create({
199 commandId: CommandId.make(`command:seed-project:${input.threadId}`),
200 projectId: input.projectId,
201 title: "Delegated completion delivery",
202 workspaceRoot: `/workspace/${input.projectId}`,
203 });
204
205 yield* orchestrator.dispatch({
206 type: "thread.create",
207 createdBy: "user",
208 creationSource: "web",
209 commandId: CommandId.make(`command:seed-create:${input.threadId}`),
210 threadId: input.threadId,
211 projectId: input.projectId,
212 title: "Delegated completion delivery",
213 modelSelection,
214 runtimeMode: "full-access",
215 interactionMode: "default",
216 branch: null,
217 worktreePath: null,
218 });
219
220 yield* eventSink.write({
221 commandId: CommandId.make(`command:seed-projection:${input.threadId}`),
222 events: [
223 {
224 id: EventId.make(`event:seed-provider-thread:${input.threadId}`),
225 type: "provider-thread.updated",
226 threadId: input.threadId,
227 driver,
228 providerInstanceId: modelSelection.instanceId,
229 occurredAt: input.now,
230 payload: {
231 id: providerThreadId,
232 driver,
233 providerInstanceId: modelSelection.instanceId,
234 providerSessionId: null,
235 appThreadId: input.threadId,
236 ownerNodeId: input.rootNodeId,
237 nativeThreadRef: {
238 driver,
239 nativeId: `native:${input.threadId}`,
240 strength: "strong",
241 },
242 nativeConversationHeadRef: null,
243 status: "active",
244 firstRunOrdinal: 1,
245 lastRunOrdinal: 1,
246 handoffIds: [],
247 forkedFrom: null,
248 createdAt: input.now,
249 updatedAt: input.now,
250 },
251 },
252 {
253 id: EventId.make(`event:seed-run:${input.threadId}`),
254 type: "run.updated",
255 threadId: input.threadId,
256 runId: input.runId,
257 nodeId: input.rootNodeId,
258 providerInstanceId: modelSelection.instanceId,
259 occurredAt: input.now,
260 payload: {
261 id: input.runId,
262 threadId: input.threadId,
263 ordinal: 1,
264 providerInstanceId: modelSelection.instanceId,
265 modelSelection,
266 providerThreadId,
267 userMessageId: MessageId.make(`message:seed-user:${input.threadId}`),
268 rootNodeId: input.rootNodeId,
269 activeAttemptId: null,
270 status: input.parentRunStatus ?? "running",
271 requestedAt: input.now,
272 startedAt: input.now,
273 completedAt: input.parentRunStatus === "completed" ? input.now : null,
274 checkpointId: null,
275 contextHandoffId: null,
276 delegatedCompletion: {
277 disposition: "open",
278 nextGeneration: 2,
279 delivery:
280 input.deliveryTaskIds === undefined
281 ? null
282 : {
283 generation: 1,
284 messageId: MessageId.make(`message:delegated-delivery:${input.threadId}`),
285 taskIds: input.deliveryTaskIds,
286 },
287 },
288 },
289 },
290 ...[
291 { id: input.taskId, deliveryState: input.deliveryState },
292 ...(input.siblingTasks ?? []),
293 ].map((task) => ({
294 id: EventId.make(`event:seed-task:${input.threadId}:${task.id}`),
295 type: "subagent.updated" as const,
296 threadId: input.threadId,
297 runId: input.runId,
298 nodeId: task.id,
299 driver,
300 providerInstanceId: modelSelection.instanceId,
301 occurredAt: input.now,
302 payload: {
303 id: task.id,
304 threadId: input.threadId,
305 runId: input.runId,
306 parentNodeId: input.rootNodeId,
307 origin: "app_owned" as const,
308 createdBy: "agent" as const,
309 driver,
310 providerInstanceId: modelSelection.instanceId,
311 providerThreadId: null,
312 childThreadId: null,
313 nativeTaskRef: null,
314 prompt: "Inspect the delivered ownership edge.",
315 title: null,
316 model: null,
317 completionWake: input.completionWake ?? "settled_only",
318 completionDelivery: {
319 state: task.deliveryState,
320 observedByRunId: task.deliveryState === "acknowledged" ? input.runId : null,
321 },
322 status: "completed" as const,
323 result: "child finished",
324 startedAt: input.now,
325 completedAt: input.now,
326 updatedAt: input.now,
327 },
328 })),
329 ],
330 });
331 });
332
333 it.layer(layerTest)("delegated completion delivery repairs", (it) => {
334 it.effect(
335 "interrupting a completion wake disposes both cohorts without reviving them on late results",
336 () =>
337 Effect.gen(function* () {
338 const orchestrator = yield* Orchestrator.OrchestratorV2;
339 const sink = yield* EventSink.EventSinkV2;
340 const now = yield* DateTime.now;
341 const threadId = ThreadId.make("stop-wake-parent");
342 const originRunId = RunId.make("stop-wake-origin");
343 const incomingTaskId = NodeId.make("stop-wake-incoming");
344 yield* seedParentWithTerminalTask({
345 threadId,
346 projectId: ProjectId.make("stop-wake-project"),
347 runId: originRunId,
348 rootNodeId: NodeId.make("stop-wake-origin-root"),
349 taskId: incomingTaskId,
350 deliveryState: "claimed",
351 deliveryTaskIds: [incomingTaskId],
352 completionWake: "always",
353 now,
354 });
355 const origin = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
356 yield* sink.write({
357 events: [
358 {
359 ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
360 payload: { ...origin, status: "completed", completedAt: now },
361 },
362 ],
363 });
364 const delivery = origin.delegatedCompletion!.delivery!;
365 yield* orchestrator.dispatch({
366 type: "message.dispatch",
367 createdBy: "agent",
368 creationSource: "server",
369 commandId: CommandId.make("stop-wake-start"),
370 threadId,
371 messageId: delivery.messageId,
372 text: "Delegated task completed",
373 attachments: [],
374 modelSelection,
375 dispatchMode: { type: "queue_after_active" },
376 delegatedCompletion: {
377 parentRunId: originRunId,
378 generation: delivery.generation,
379 taskIds: delivery.taskIds,
380 },
381 });
382 yield* orchestrator.resumeQueuedRuns;
383 const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
384 (run) => run.userMessageId === delivery.messageId,
385 )!;
386 yield* orchestrator.dispatch({
387 type: "delegated_task.request",
388 createdBy: "agent",
389 creationSource: "mcp",
390 commandId: CommandId.make("stop-wake-delegate"),
391 parentThreadId: threadId,
392 parentRunId: wake.id,
393 parentNodeId: wake.rootNodeId!,
394 task: "Do the next round",
395 modelSelection,
396 runtimeMode: "full-access",
397 interactionMode: "default",
398 completionWake: "always",
399 });
400 const delegated = (yield* orchestrator.getThreadProjection(threadId)).subagents.find(
401 (task) => task.runId === wake.id,
402 )!;
403 yield* orchestrator.dispatch({
404 type: "run.interrupt",
405 commandId: CommandId.make("stop-wake-interrupt"),
406 threadId,
407 runId: wake.id,
408 });
409 const stopped = yield* orchestrator.getThreadProjection(threadId);
410 for (const runId of [originRunId, wake.id]) {
411 const run = stopped.runs.find((row) => row.id === runId)!;
412 assert.equal(run.delegatedCompletion?.disposition, "stopped");
413 assert.isNull(run.delegatedCompletion?.delivery);
414 }
415 assert.equal(stopped.runs.find((run) => run.id === wake.id)?.status, "interrupted");
416 for (const taskId of [incomingTaskId, delegated.id]) {
417 assert.equal(
418 stopped.subagents.find((task) => task.id === taskId)?.completionDelivery?.state,
419 "disposed",
420 );
421 }
422 assert.equal(
423 stopped.subagents.find((task) => task.id === incomingTaskId)?.result,
424 "child finished",
425 );
426
427 yield* orchestrator.dispatch({
428 type: "message.dispatch",
429 createdBy: "user",
430 creationSource: "web",
431 commandId: CommandId.make("stop-wake-newer-work"),
432 threadId,
433 messageId: MessageId.make("stop-wake-newer-work"),
434 text: "Start unrelated work",
435 attachments: [],
436 modelSelection,
437 dispatchMode: { type: "queue_after_active" },
438 });
439 yield* orchestrator.resumeQueuedRuns;
440 const newer = yield* orchestrator.getThreadProjection(threadId);
441 assert.equal(newer.runs.at(-1)?.status, "starting");
442
443 const childThreadId = delegated.childThreadId!;
444 const childRun = (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]!;
445 const afterSequence = yield* sink.latestSequence();
446 yield* sink.write({
447 events: [
448 {
449 ...runEvent({
450 threadId: childThreadId,
451 runId: childRun.id,
452 ordinal: childRun.ordinal,
453 status: "completed",
454 now,
455 }),
456 payload: { ...childRun, status: "completed", completedAt: now },
457 },
458 ],
459 });
460 yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
461 Stream.filter(
462 (stored) =>
463 stored.event.type === "subagent.updated" &&
464 stored.event.payload.id === delegated.id &&
465 stored.event.payload.status === "completed",
466 ),
467 Stream.take(1),
468 Stream.runDrain,
469 );
470 const finished = yield* orchestrator.getThreadProjection(threadId);
471 const lateTask = finished.subagents.find((task) => task.id === delegated.id)!;
472 assert.equal(lateTask.status, "completed");
473 assert.isNotNull(lateTask.result);
474 assert.equal(lateTask.completionDelivery?.state, "disposed");
475 assert.deepEqual(finished.runs, newer.runs);
476 assert.deepEqual(finished.messages, newer.messages);
477 }),
478 );
479
480 it.effect("acceptance batches pending siblings without acknowledging their results", () =>
481 Effect.gen(function* () {
482 const orchestrator = yield* Orchestrator.OrchestratorV2;
483 const sink = yield* EventSink.EventSinkV2;
484 const now = yield* DateTime.now;
485 const threadId = ThreadId.make("mailbox-batch");
486 const runId = RunId.make("mailbox-parent");
487 const taskId = NodeId.make("mailbox-first");
488 const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
489 yield* seedParentWithTerminalTask({
490 threadId,
491 runId,
492 projectId: ProjectId.make("mailbox-project"),
493 rootNodeId: NodeId.make("mailbox-root"),
494 taskId,
495 deliveryState: "claimed",
496 completionWake: "always",
497 deliveryTaskIds: [taskId],
498 now,
499 });
500 const projection = yield* orchestrator.getThreadProjection(threadId);
501 const task = projection.subagents[0]!;
502 const pendingIds = [NodeId.make("mailbox-second"), NodeId.make("mailbox-third")];
503 yield* sink.write({
504 events: [
505 {
506 id: EventId.make("mailbox-message"),
507 type: "message.updated",
508 threadId,
509 runId,
510 occurredAt: now,
511 payload: {
512 id: messageId,
513 threadId,
514 runId,
515 nodeId: task.parentNodeId,
516 role: "user",
517 text: "Background task finished",
518 attachments: [],
519 streaming: false,
520 createdBy: "agent",
521 creationSource: "server",
522 createdAt: now,
523 updatedAt: now,
524 delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
525 },
526 },
527 ...pendingIds.map((id) => ({
528 id: EventId.make(`event:${id}`),
529 type: "subagent.updated" as const,
530 threadId,
531 runId,
532 nodeId: id,
533 occurredAt: now,
534 payload: {
535 ...task,
536 id,
537 completionDelivery: { state: "pending" as const, observedByRunId: null },
538 },
539 })),
540 ],
541 });
542 yield* orchestrator.dispatch({
543 type: "notification.delivery.accept",
544 commandId: CommandId.make("accept-first"),
545 threadId,
546 messageId,
547 });
548 const accepted = yield* orchestrator.getThreadProjection(threadId);
549 assert.equal(
550 accepted.subagents.find((row) => row.id === taskId)?.completionDelivery?.state,
551 "delivered",
552 );
553 const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion;
554 assert.deepEqual(cohort?.delivery?.taskIds, pendingIds);
555 assert.equal(cohort?.delivery?.generation, 2);
556 for (const id of pendingIds) {
557 assert.deepEqual(accepted.subagents.find((row) => row.id === id)?.completionDelivery, {
558 state: "claimed",
559 observedByRunId: null,
560 });
561 }
562 yield* orchestrator.dispatch({
563 type: "notification.delivery.accept",
564 commandId: CommandId.make("repeat-old-acceptance"),
565 threadId,
566 messageId,
567 });
568 const duplicate = yield* orchestrator.getThreadProjection(threadId);
569 assert.deepEqual(duplicate.runs.find((row) => row.id === runId)?.delegatedCompletion, cohort);
570 }),
571 );
572
573 it.effect("acceptance batches a settled_only sibling once its spawning run ended", () =>
574 Effect.gen(function* () {
575 const orchestrator = yield* Orchestrator.OrchestratorV2;
576 const sink = yield* EventSink.EventSinkV2;
577 const now = yield* DateTime.now;
578 const threadId = ThreadId.make("mailbox-mixed");
579 const runId = RunId.make("mailbox-mixed-parent");
580 const taskId = NodeId.make("mailbox-mixed-always");
581 const siblingId = NodeId.make("mailbox-mixed-settled-only");
582 const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
583 yield* seedParentWithTerminalTask({
584 threadId,
585 runId,
586 projectId: ProjectId.make("mailbox-mixed-project"),
587 rootNodeId: NodeId.make("mailbox-mixed-root"),
588 taskId,
589 deliveryState: "claimed",
590 completionWake: "always",
591 deliveryTaskIds: [taskId],
592 now,
593 });
594 const projection = yield* orchestrator.getThreadProjection(threadId);
595 const task = projection.subagents[0]!;
596 const spawningRun = projection.runs.find((row) => row.id === runId)!;
597 yield* sink.write({
598 events: [
599 {
600 ...runEvent({ threadId, runId, ordinal: 1, status: "cancelled", now }),
601 payload: { ...spawningRun, status: "cancelled" as const, completedAt: now },
602 },
603 runEvent({
604 threadId,
605 runId: RunId.make("mailbox-mixed-continuation"),
606 ordinal: 2,
607 status: "running",
608 now,
609 }),
610 {
611 id: EventId.make("mailbox-mixed-message"),
612 type: "message.updated",
613 threadId,
614 runId,
615 occurredAt: now,
616 payload: {
617 id: messageId,
618 threadId,
619 runId,
620 nodeId: task.parentNodeId,
621 role: "user",
622 text: "Background task finished",
623 attachments: [],
624 streaming: false,
625 createdBy: "agent",
626 creationSource: "server",
627 createdAt: now,
628 updatedAt: now,
629 delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
630 },
631 },
632 {
633 id: EventId.make(`event:${siblingId}`),
634 type: "subagent.updated" as const,
635 threadId,
636 runId,
637 nodeId: siblingId,
638 occurredAt: now,
639 payload: {
640 ...task,
641 id: siblingId,
642 completionWake: "settled_only" as const,
643 completionDelivery: { state: "pending" as const, observedByRunId: null },
644 },
645 },
646 ],
647 });
648 yield* orchestrator.dispatch({
649 type: "notification.delivery.accept",
650 commandId: CommandId.make("accept-mixed"),
651 threadId,
652 messageId,
653 });
654 const accepted = yield* orchestrator.getThreadProjection(threadId);
655 const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion;
656 assert.deepEqual(cohort?.delivery?.taskIds, [siblingId]);
657 assert.equal(
658 accepted.subagents.find((row) => row.id === siblingId)?.completionDelivery?.state,
659 "claimed",
660 );
661 }),
662 );
663
664 it.effect("builds completion text and metadata from the same live cohort", () =>
665 Effect.gen(function* () {
666 const orchestrator = yield* Orchestrator.OrchestratorV2;
667 const now = yield* DateTime.now;
668 const threadId = ThreadId.make("thread:delegated-delivery-live-cohort");
669 const projectId = ProjectId.make("project:delegated-delivery-live-cohort");
670 const runId = RunId.make("run:delegated-delivery-live-cohort");
671 const rootNodeId = NodeId.make("node:delegated-delivery-live-cohort-root");
672 const firstTaskId = NodeId.make("node:delegated-delivery-live-cohort-first");
673 const secondTaskId = NodeId.make("node:delegated-delivery-live-cohort-second");
674 const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
675
676 yield* seedParentWithTerminalTask({
677 threadId,
678 projectId,
679 runId,
680 rootNodeId,
681 taskId: firstTaskId,
682 deliveryState: "claimed",
683 completionWake: "always",
684 deliveryTaskIds: [firstTaskId, secondTaskId],
685 now,
686 });
687
688 yield* orchestrator.dispatch({
689 type: "message.dispatch",
690 commandId: CommandId.make("command:delegated-delivery-live-cohort"),
691 threadId,
692 messageId,
693 text: `Delegated task ${firstTaskId} reached a terminal state.`,
694 attachments: [],
695 dispatchMode: { type: "queue_after_active" },
696 createdBy: "agent",
697 creationSource: "server",
698 delegatedCompletion: {
699 parentRunId: runId,
700 generation: 1,
701 taskIds: [firstTaskId],
702 },
703 });
704
705 const projection = yield* orchestrator.getThreadProjection(threadId);
706 const message = projection.messages.find((candidate) => candidate.id === messageId);
707 assert.deepEqual(message?.delegatedCompletion?.taskIds, [firstTaskId, secondTaskId]);
708 assert.include(message?.text ?? "", String(firstTaskId));
709 assert.include(message?.text ?? "", String(secondTaskId));
710 assert.include(message?.text ?? "", "task_status");
711 assert.include(message?.text ?? "", "child finished");
712 }),
713 );
714
715 it.effect("counts already delivered siblings in the completion notice", () =>
716 Effect.gen(function* () {
717 const orchestrator = yield* Orchestrator.OrchestratorV2;
718 const now = yield* DateTime.now;
719 const threadId = ThreadId.make("thread:delegated-delivery-notice-count");
720 const runId = RunId.make("run:delegated-delivery-notice-count");
721 const firstTaskId = NodeId.make("node:delegated-delivery-notice-count-first");
722 const secondTaskId = NodeId.make("node:delegated-delivery-notice-count-second");
723
724 yield* seedParentWithTerminalTask({
725 threadId,
726 projectId: ProjectId.make("project:delegated-delivery-notice-count"),
727 runId,
728 rootNodeId: NodeId.make("node:delegated-delivery-notice-count-root"),
729 taskId: firstTaskId,
730 deliveryState: "claimed",
731 completionWake: "always",
732 deliveryTaskIds: [firstTaskId, secondTaskId],
733 parentRunStatus: "completed",
734 siblingTasks: [
735 { id: secondTaskId, deliveryState: "claimed" },
736 {
737 id: NodeId.make("node:delegated-delivery-notice-count-earlier"),
738 deliveryState: "delivered",
739 },
740 ],
741 now,
742 });
743
744 yield* orchestrator.dispatch({
745 type: "message.dispatch",
746 commandId: CommandId.make("command:delegated-delivery-notice-count"),
747 threadId,
748 messageId: MessageId.make(`message:delegated-delivery:${threadId}`),
749 text: "Delegated tasks reached a terminal state.",
750 attachments: [],
751 dispatchMode: { type: "queue_after_active" },
752 createdBy: "agent",
753 creationSource: "server",
754 delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [firstTaskId] },
755 });
756
757 const projection = yield* orchestrator.getThreadProjection(threadId);
758 const notice = projection.turnItems.find((item) => item.type === "notification");
759 assert.match(
760 notice?.type === "notification" ? notice.summary : "",
761 /^2 of 3 delegated tasks finished:/,
762 );
763 }),
764 );
765
766 it.effect.each([
767 { name: "no result is waiting", delivery: "none", compacts: true },
768 { name: "a result is waiting to be sent", delivery: "pending", compacts: false },
769 { name: "a result was just sent", delivery: "sent", compacts: false },
770 ] as const)("compacts before the cache expires only when $name", ({ delivery, compacts }) =>
771 Effect.gen(function* () {
772 const orchestrator = yield* Orchestrator.OrchestratorV2;
773 const now = yield* DateTime.now;
774 const threadId = ThreadId.make(`thread:idle-compaction-delivery-${delivery}`);
775 const runId = RunId.make(`run:idle-compaction-delivery-${delivery}`);
776 const taskId = NodeId.make(`node:idle-compaction-delivery-${delivery}-task`);
777
778 yield* seedParentWithTerminalTask({
779 threadId,
780 projectId: ProjectId.make(`project:idle-compaction-delivery-${delivery}`),
781 runId,
782 rootNodeId: NodeId.make(`node:idle-compaction-delivery-${delivery}-root`),
783 taskId,
784 deliveryState: delivery === "none" ? "delivered" : "claimed",
785 completionWake: "always",
786 parentRunStatus: "completed",
787 ...(delivery === "none" ? {} : { deliveryTaskIds: [taskId] }),
788 now,
789 });
790 if (delivery === "sent") {
791 yield* orchestrator.dispatch({
792 type: "message.dispatch",
793 commandId: CommandId.make(`command:idle-compaction-delivery-${delivery}`),
794 threadId,
795 messageId: MessageId.make(`message:delegated-delivery:${threadId}`),
796 text: `Delegated task ${taskId} reached a terminal state.`,
797 attachments: [],
798 dispatchMode: { type: "queue_after_active" },
799 createdBy: "agent",
800 creationSource: "server",
801 delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
802 });
803 }
804
805 const compactionId = MessageId.make(`idle-compact:${threadId}:${runId}`);
806 yield* orchestrator.dispatch({
807 type: "message.dispatch",
808 commandId: CommandId.make(`server:idle-compact:${threadId}:${runId}`),
809 threadId,
810 messageId: compactionId,
811 text: "/compact",
812 attachments: [],
813 createdBy: "agent",
814 creationSource: "server",
815 idleCompaction: "beforeCacheExpires",
816 dispatchMode: { type: "start_immediately" },
817 });
818
819 const projection = yield* orchestrator.getThreadProjection(threadId);
820 assert.equal(
821 projection.messages.some((message) => message.id === compactionId),
822 compacts,
823 );
824 }),
825 );
826
827 it.effect.each([
828 {
829 name: "oversized UTF-8 result",
830 first: "界".repeat(6_000),
831 second: "small result",
832 inlineFirst: false,
833 inlineSecond: true,
834 },
835 {
836 name: "combined results",
837 first: "a".repeat(10_000),
838 second: "b".repeat(10_000),
839 inlineFirst: true,
840 inlineSecond: false,
841 },
842 ])("uses task_status for $name beyond the inline budget", (testCase) =>
843 Effect.gen(function* () {
844 const orchestrator = yield* Orchestrator.OrchestratorV2;
845 const sink = yield* EventSink.EventSinkV2;
846 const now = yield* DateTime.now;
847 const threadId = ThreadId.make(`thread:inline-budget:${testCase.name}`);
848 const runId = RunId.make(`run:inline-budget:${testCase.name}`);
849 const firstTaskId = NodeId.make(`node:inline-first:${testCase.name}`);
850 const secondTaskId = NodeId.make(`node:inline-second:${testCase.name}`);
851 const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
852 yield* seedParentWithTerminalTask({
853 threadId,
854 projectId: ProjectId.make(`project:inline-budget:${testCase.name}`),
855 runId,
856 rootNodeId: NodeId.make(`node:inline-budget:${testCase.name}`),
857 taskId: firstTaskId,
858 deliveryState: "claimed",
859 completionWake: "always",
860 deliveryTaskIds: [firstTaskId, secondTaskId],
861 now,
862 });
863 const parent = yield* orchestrator.getThreadProjection(threadId);
864 const task = parent.subagents[0]!;
865 yield* sink.write({
866 events: [
867 { id: firstTaskId, result: testCase.first, status: "failed" as const },
868 { id: secondTaskId, result: testCase.second, status: "cancelled" as const },
869 ].map(({ id, result, status }) => ({
870 id: EventId.make(`event:inline-budget:${id}`),
871 type: "subagent.updated" as const,
872 threadId,
873 runId,
874 nodeId: id,
875 occurredAt: now,
876 payload: {
877 ...task,
878 id,
879 result,
880 status,
881 title: "Inspect regression",
882 childThreadId: ThreadId.make(`thread:inline-budget-child:${id}`),
883 },
884 })),
885 });
886 yield* orchestrator.dispatch({
887 type: "message.dispatch",
888 commandId: CommandId.make(`command:inline-budget:${testCase.name}`),
889 threadId,
890 messageId,
891 text: "Delegated tasks completed.",
892 attachments: [],
893 dispatchMode: { type: "queue_after_active" },
894 createdBy: "agent",
895 creationSource: "server",
896 delegatedCompletion: {
897 parentRunId: runId,
898 generation: 1,
899 taskIds: [firstTaskId, secondTaskId],
900 },
901 });
902 const projection = yield* orchestrator.getThreadProjection(threadId);
903 const message = projection.messages.find((candidate) => candidate.id === messageId)!;
904 assert.equal(message.text.includes(testCase.first), testCase.inlineFirst);
905 assert.equal(message.text.includes(testCase.second), testCase.inlineSecond);
906 const deferredTaskId = testCase.inlineFirst ? secondTaskId : firstTaskId;
907 assert.include(
908 message.text,
909 `Delegated task ${deferredTaskId} (Inspect regression) in child thread thread:inline-budget-child:${deferredTaskId} [${testCase.inlineFirst ? "cancelled" : "failed"}]`,
910 );
911 assert.include(message.text, `Use task_status with taskId ${deferredTaskId}`);
912 assert.deepEqual(message.delegatedCompletion?.taskIds, [firstTaskId, secondTaskId]);
913 assert.equal(
914 projection.subagents.find((candidate) => candidate.id === deferredTaskId)?.result,
915 testCase.inlineFirst ? testCase.second : testCase.first,
916 );
917 }),
918 );
919
920 it.effect("does not re-offer when wake-policy upgrades after delivered ownership settled", () =>
921 Effect.gen(function* () {
922 const orchestrator = yield* Orchestrator.OrchestratorV2;
923 const now = yield* DateTime.now;
924 const threadId = ThreadId.make("thread:delegated-delivery-a1");
925 const projectId = ProjectId.make("project:delegated-delivery-a1");
926 const runId = RunId.make("run:delegated-delivery-a1");
927 const rootNodeId = NodeId.make("node:delegated-delivery-a1-root");
928 const taskId = NodeId.make("node:delegated-delivery-a1-task");
929
930 yield* seedParentWithTerminalTask({
931 threadId,
932 projectId,
933 runId,
934 rootNodeId,
935 taskId,
936 deliveryState: "delivered",
937 completionWake: "settled_only",
938 now,
939 });
940
941 const upgrade = yield* orchestrator.dispatch({
942 type: "delegated_task.wake-policy",
943 commandId: CommandId.make("command:delegated-delivery-a1:wake-policy"),
944 parentThreadId: threadId,
945 taskId,
946 completionWake: "always",
947 });
948
949 const projection = yield* orchestrator.getThreadProjection(threadId);
950 const task = projection.subagents.find((candidate) => candidate.id === taskId);
951 const parentRun = projection.runs.find((candidate) => candidate.id === runId);
952
953 assert.equal(task?.completionWake, "always");
954 assert.deepEqual(task?.completionDelivery, {
955 state: "delivered",
956 observedByRunId: null,
957 });
958 assert.deepEqual(parentRun?.delegatedCompletion, {
959 disposition: "open",
960 nextGeneration: 2,
961 delivery: null,
962 });
963 assert.isFalse(
964 upgrade.storedEvents.some(
965 (stored) =>
966 stored.event.type === "subagent.updated" &&
967 stored.event.payload.id === taskId &&
968 stored.event.payload.completionDelivery?.state === "claimed",
969 ),
970 );
971 assert.isFalse(
972 upgrade.storedEvents.some(
973 (stored) =>
974 stored.event.type === "run.updated" &&
975 stored.event.payload.id === runId &&
976 stored.event.payload.delegatedCompletion?.delivery !== null &&
977 stored.event.payload.delegatedCompletion?.delivery !== undefined,
978 ),
979 );
980 }),
981 );
982
983 it.effect(
984 "treats repeated acknowledge and dispose with distinct command IDs as successful no-ops",
985 () =>
986 Effect.gen(function* () {
987 const orchestrator = yield* Orchestrator.OrchestratorV2;
988 const now = yield* DateTime.now;
989 const threadId = ThreadId.make("thread:delegated-delivery-a2");
990 const projectId = ProjectId.make("project:delegated-delivery-a2");
991 const runId = RunId.make("run:delegated-delivery-a2");
992 const rootNodeId = NodeId.make("node:delegated-delivery-a2-root");
993 const taskId = NodeId.make("node:delegated-delivery-a2-task");
994
995 yield* seedParentWithTerminalTask({
996 threadId,
997 projectId,
998 runId,
999 rootNodeId,
1000 taskId,
1001 deliveryState: "delivered",
1002 completionWake: "always",
1003 now,
1004 });
1005
1006 const firstAck = yield* orchestrator.dispatch({
1007 type: "delegated_task.completion-delivery.acknowledge",
1008 commandId: CommandId.make("command:delegated-delivery-a2:ack-task-status"),
1009 parentThreadId: threadId,
1010 taskId,
1011 observedByRunId: runId,
1012 });
1013 const secondAck = yield* orchestrator.dispatch({
1014 type: "delegated_task.completion-delivery.acknowledge",
1015 commandId: CommandId.make("command:delegated-delivery-a2:ack-thread-read"),
1016 parentThreadId: threadId,
1017 taskId,
1018 observedByRunId: runId,
1019 });
1020
1021 const firstAckTask = firstAck.storedEvents.find(
1022 (stored) =>
1023 stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
1024 );
1025 const secondAckTask = secondAck.storedEvents.find(
1026 (stored) =>
1027 stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
1028 );
1029 assert.isDefined(firstAckTask);
1030 assert.isDefined(secondAckTask);
1031 if (
1032 firstAckTask?.event.type !== "subagent.updated" ||
1033 secondAckTask?.event.type !== "subagent.updated"
1034 ) {
1035 return yield* Effect.die(new Error("Acknowledge events missing."));
1036 }
1037 assert.equal(firstAckTask.event.payload.completionDelivery?.state, "acknowledged");
1038 assert.equal(secondAckTask.event.payload.completionDelivery?.state, "acknowledged");
1039 assert.deepEqual(
1040 secondAckTask.event.payload.completionDelivery,
1041 firstAckTask.event.payload.completionDelivery,
1042 );
1043 assert.deepEqual(
1044 secondAckTask.event.payload.updatedAt,
1045 firstAckTask.event.payload.updatedAt,
1046 );
1047 assert.equal(secondAck.storedEvents.length, 1);
1048
1049 const afterAck = yield* orchestrator.getThreadProjection(threadId);
1050 assert.deepEqual(
1051 afterAck.subagents.find((candidate) => candidate.id === taskId)?.completionDelivery,
1052 {
1053 state: "acknowledged",
1054 observedByRunId: runId,
1055 },
1056 );
1057
1058 const firstDispose = yield* orchestrator.dispatch({
1059 type: "delegated_task.completion-delivery.dispose",
1060 commandId: CommandId.make("command:delegated-delivery-a2:dispose-task-status"),
1061 parentThreadId: threadId,
1062 taskId,
1063 });
1064 const secondDispose = yield* orchestrator.dispatch({
1065 type: "delegated_task.completion-delivery.dispose",
1066 commandId: CommandId.make("command:delegated-delivery-a2:dispose-thread-read"),
1067 parentThreadId: threadId,
1068 taskId,
1069 });
1070
1071 const firstDisposeTask = firstDispose.storedEvents.find(
1072 (stored) =>
1073 stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
1074 );
1075 const secondDisposeTask = secondDispose.storedEvents.find(
1076 (stored) =>
1077 stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
1078 );
1079 assert.isDefined(firstDisposeTask);
1080 assert.isDefined(secondDisposeTask);
1081 if (
1082 firstDisposeTask?.event.type !== "subagent.updated" ||
1083 secondDisposeTask?.event.type !== "subagent.updated"
1084 ) {
1085 return yield* Effect.die(new Error("Dispose events missing."));
1086 }
1087 assert.equal(firstDisposeTask.event.payload.completionDelivery?.state, "disposed");
1088 assert.equal(secondDisposeTask.event.payload.completionDelivery?.state, "disposed");
1089 assert.deepEqual(
1090 secondDisposeTask.event.payload.completionDelivery,
1091 firstDisposeTask.event.payload.completionDelivery,
1092 );
1093 assert.equal(secondDispose.storedEvents.length, 1);
1094
1095 const afterDispose = yield* orchestrator.getThreadProjection(threadId);
1096 assert.deepEqual(
1097 afterDispose.subagents.find((candidate) => candidate.id === taskId)?.completionDelivery,
1098 {
1099 state: "disposed",
1100 observedByRunId: null,
1101 },
1102 );
1103
1104 const acknowledgeAfterDispose = yield* orchestrator.dispatch({
1105 type: "delegated_task.completion-delivery.acknowledge",
1106 commandId: CommandId.make("command:delegated-delivery-a2:ack-after-dispose"),
1107 parentThreadId: threadId,
1108 taskId,
1109 observedByRunId: runId,
1110 });
1111 const acknowledgedTask = acknowledgeAfterDispose.storedEvents.find(
1112 (stored) => stored.event.type === "subagent.updated",
1113 );
1114 if (acknowledgedTask?.event.type !== "subagent.updated") {
1115 return yield* Effect.die(new Error("Acknowledge-after-dispose event missing."));
1116 }
1117 assert.deepEqual(acknowledgedTask.event.payload.completionDelivery, {
1118 state: "disposed",
1119 observedByRunId: null,
1120 });
1121 assert.equal(acknowledgeAfterDispose.storedEvents.length, 1);
1122
1123 const afterStaleAcknowledge = yield* orchestrator.getThreadProjection(threadId);
1124 assert.deepEqual(
1125 afterStaleAcknowledge.subagents.find((candidate) => candidate.id === taskId)
1126 ?.completionDelivery,
1127 {
1128 state: "disposed",
1129 observedByRunId: null,
1130 },
1131 );
1132 }),
1133 );
1134
1135 it.effect("acknowledging a queued wake's results rewrites it, then cancels it", () =>
1136 Effect.gen(function* () {
1137 const orchestrator = yield* Orchestrator.OrchestratorV2;
1138 const now = yield* DateTime.now;
1139 const threadId = ThreadId.make("queued-wake-ack-parent");
1140 const runId = RunId.make("queued-wake-ack-origin");
1141 const taskId = NodeId.make("queued-wake-ack-first");
1142 const siblingId = NodeId.make("queued-wake-ack-sibling");
1143 yield* seedParentWithTerminalTask({
1144 threadId,
1145 projectId: ProjectId.make("queued-wake-ack-project"),
1146 runId,
1147 rootNodeId: NodeId.make("queued-wake-ack-root"),
1148 taskId,
1149 deliveryState: "claimed",
1150 deliveryTaskIds: [taskId, siblingId],
1151 siblingTasks: [{ id: siblingId, deliveryState: "claimed" }],
1152 completionWake: "always",
1153 now,
1154 });
1155 const delivery = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!
1156 .delegatedCompletion!.delivery!;
1157 yield* orchestrator.dispatch({
1158 type: "message.dispatch",
1159 createdBy: "agent",
1160 creationSource: "server",
1161 commandId: CommandId.make("queued-wake-ack-wake"),
1162 threadId,
1163 messageId: delivery.messageId,
1164 text: "Delegated tasks completed",
1165 attachments: [],
1166 modelSelection,
1167 dispatchMode: { type: "queue_after_active" },
1168 delegatedCompletion: {
1169 parentRunId: runId,
1170 generation: delivery.generation,
1171 taskIds: delivery.taskIds,
1172 },
1173 });
1174 const acknowledge = (id: NodeId) =>
1175 orchestrator.dispatch({
1176 type: "delegated_task.completion-delivery.acknowledge",
1177 commandId: CommandId.make(`queued-wake-ack:${id}`),
1178 parentThreadId: threadId,
1179 taskId: id,
1180 observedByRunId: runId,
1181 });
1182
1183 yield* acknowledge(taskId);
1184 const rewritten = yield* orchestrator.getThreadProjection(threadId);
1185 const wake = rewritten.runs.find((run) => run.userMessageId === delivery.messageId)!;
1186 assert.equal(wake.status, "queued");
1187 const message = rewritten.messages.find((row) => row.id === delivery.messageId);
1188 assert.deepEqual(message?.delegatedCompletion?.taskIds, [siblingId]);
1189 assert.equal(message?.text, `Delegated task ${siblingId} [completed]:\nchild finished`);
1190
1191 yield* acknowledge(siblingId);
1192 const cancelled = yield* orchestrator.getThreadProjection(threadId);
1193 assert.equal(cancelled.runs.find((run) => run.id === wake.id)?.status, "cancelled");
1194 assert.equal(
1195 cancelled.nodes.find((node) => node.id === wake.rootNodeId)?.status,
1196 "cancelled",
1197 );
1198 assert.isNull(cancelled.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery);
1199 }),
1200 );
1201
1202 it.effect.each([
1203 { command: "acknowledge", before: "running", after: "failed", capture: false },
1204 { command: "acknowledge", before: "waiting", after: "completed", capture: true },
1205 { command: "accept", before: "waiting", after: "completed", capture: true },
1206 { command: "wake-policy", before: "running", after: "completed", capture: true },
1207 ] as const)(
1208 "keeps a $after run that settled between $command's read and commit",
1209 ({ command, before, after, capture }) =>
1210 Effect.gen(function* () {
1211 const orchestrator = yield* Orchestrator.OrchestratorV2;
1212 const sink = yield* EventSink.EventSinkV2;
1213 const now = yield* DateTime.now;
1214 const race = `${command}-settle-race-${after}`;
1215 const threadId = ThreadId.make(`thread:${race}`);
1216 const runId = RunId.make(`run:${race}`);
1217 const taskId = NodeId.make(`node:${race}-task`);
1218 const checkpointId = CheckpointId.make(`checkpoint:${race}`);
1219 const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
1220 const commandType = {
1221 acknowledge: "delegated_task.completion-delivery.acknowledge",
1222 accept: "notification.delivery.accept",
1223 "wake-policy": "delegated_task.wake-policy",
1224 }[command];
1225 yield* seedParentWithTerminalTask({
1226 threadId,
1227 projectId: ProjectId.make(`project:${race}`),
1228 runId,
1229 rootNodeId: NodeId.make(`node:${race}-root`),
1230 taskId,
1231 ...(command === "wake-policy"
1232 ? { deliveryState: "pending" }
1233 : { deliveryState: "claimed", deliveryTaskIds: [taskId] }),
1234 now,
1235 });
1236 const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
1237 yield* sink.write({
1238 events: [
1239 {
1240 id: EventId.make(`event:${race}-before`),
1241 type: "run.updated",
1242 threadId,
1243 runId,
1244 occurredAt: now,
1245 payload: { ...parent, status: before },
1246 },
1247 {
1248 id: EventId.make(`event:${race}-message`),
1249 type: "message.updated",
1250 threadId,
1251 runId,
1252 occurredAt: now,
1253 payload: {
1254 id: messageId,
1255 threadId,
1256 runId,
1257 nodeId: parent.rootNodeId,
1258 role: "user",
1259 text: "Background task finished",
1260 attachments: [],
1261 streaming: false,
1262 createdBy: "agent",
1263 creationSource: "server",
1264 createdAt: now,
1265 updatedAt: now,
1266 delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
1267 },
1268 },
1269 ],
1270 });
1271 const { delegatedCompletion: _cohort, ...settledRun } = parent;
1272 const commit = sink.commitCommand;
1273 let settled = false;
1274 let planned: OrchestrationV2Run["delegatedCompletion"];
1275 yield* Effect.acquireUseRelease(
1276 Effect.sync(() =>
1277 vi.spyOn(sink, "commitCommand").mockImplementation((input) =>
1278 Effect.gen(function* () {
1279 if (!settled && input.commandType === commandType) {
1280 settled = true;
1281 const plannedRun = input.events.find((event) => event.type === "run.updated");
1282 planned =
1283 plannedRun?.type === "run.updated"
1284 ? plannedRun.payload.delegatedCompletion
1285 : undefined;
1286 // The provider turn ends (or its checkpoint lands) after the command read the run.
1287 yield* commit({
1288 commandId: CommandId.make(`command:${race}`),
1289 threadId,
1290 commandType: capture ? "checkpoint.capture" : "provider.event",
1291 acceptedAt: now,
1292 effects: [],
1293 events: [
1294 {
1295 id: EventId.make(`event:${race}-settled`),
1296 type: "run.updated",
1297 threadId,
1298 runId,
1299 occurredAt: now,
1300 payload: {
1301 ...settledRun,
1302 status: after,
1303 completedAt: now,
1304 checkpointId: capture ? checkpointId : null,
1305 },
1306 },
1307 ],
1308 });
1309 }
1310 return yield* commit(input);
1311 }),
1312 ),
1313 ),
1314 () =>
1315 orchestrator.dispatch(
1316 command === "accept"
1317 ? {
1318 type: "notification.delivery.accept",
1319 commandId: CommandId.make(`command:${race}:accept`),
1320 threadId,
1321 messageId,
1322 }
1323 : command === "wake-policy"
1324 ? {
1325 type: "delegated_task.wake-policy",
1326 commandId: CommandId.make(`command:${race}:wake-policy`),
1327 parentThreadId: threadId,
1328 taskId,
1329 completionWake: "always",
1330 }
1331 : {
1332 type: "delegated_task.completion-delivery.acknowledge",
1333 commandId: CommandId.make(`command:${race}:task-status`),
1334 parentThreadId: threadId,
1335 taskId,
1336 observedByRunId: runId,
1337 },
1338 ),
1339 (spy) => Effect.sync(() => spy.mockRestore()),
1340 );
1341 assert.isTrue(settled);
1342 const finished = yield* orchestrator.getThreadProjection(threadId);
1343 assert.equal(
1344 finished.subagents.find((row) => row.id === taskId)?.completionDelivery?.state,
1345 { acknowledge: "acknowledged", accept: "delivered", "wake-policy": "claimed" }[command],
1346 );
1347 const run = finished.runs.find((candidate) => candidate.id === runId)!;
1348 assert.deepEqual(
1349 {
1350 status: run.status,
1351 completedAt: run.completedAt,
1352 checkpointId: run.checkpointId,
1353 delegatedCompletion: run.delegatedCompletion,
1354 },
1355 {
1356 status: after,
1357 completedAt: now,
1358 checkpointId: capture ? checkpointId : null,
1359 delegatedCompletion: planned,
1360 },
1361 );
1362 }),
1363 );
1364 });
1365
1366 const reconcileCommandId = (name: string) => CommandId.make(`command:runtime-reconcile:${name}`);
1367
1368 const parentProviderThreadId = (threadId: ThreadId) =>
1369 ProviderThreadId.make(`provider-thread:${String(threadId).replace("thread:", "")}`);
1370
1371 const runEvent = (input: {
1372 readonly threadId: ThreadId;
1373 readonly runId: RunId;
1374 readonly ordinal: number;
1375 readonly status: OrchestrationV2Run["status"];
1376 readonly now: DateTime.Utc;
1377 readonly providerThreadId?: ProviderThreadId;
1378 readonly userMessageId?: MessageId;
1379 readonly delegatedCompletion?: OrchestrationV2Run["delegatedCompletion"];
1380 }) => ({
1381 id: EventId.make(`event:${input.runId}:${input.status}`),
1382 type: "run.updated" as const,
1383 threadId: input.threadId,
1384 runId: input.runId,
1385 providerInstanceId: modelSelection.instanceId,
1386 occurredAt: input.now,
1387 payload: {
1388 id: input.runId,
1389 threadId: input.threadId,
1390 ordinal: input.ordinal,
1391 providerInstanceId: modelSelection.instanceId,
1392 modelSelection,
1393 providerThreadId: input.providerThreadId ?? null,
1394 userMessageId: input.userMessageId ?? MessageId.make(`message:${input.runId}`),
1395 rootNodeId: null,
1396 activeAttemptId: null,
1397 status: input.status,
1398 requestedAt: input.now,
1399 startedAt: input.now,
1400 completedAt: input.status === "running" ? null : input.now,
1401 checkpointId: null,
1402 contextHandoffId: null,
1403 ...(input.delegatedCompletion === undefined
1404 ? {}
1405 : { delegatedCompletion: input.delegatedCompletion }),
1406 },
1407 });
1408
1409 const seedRestartCancelledChild = (input: {
1410 readonly parentThreadId: ThreadId;
1411 readonly projectId: ProjectId;
1412 readonly parentRunId: RunId;
1413 readonly rootNodeId: NodeId;
1414 readonly name: string;
1415 readonly completionWake: "always" | "settled_only";
1416 readonly continuationPending: boolean;
1417 readonly runStatus?: "cancelled" | "completed";
1418 readonly lostWork?: OrchestrationV2Run["restartCancelledBackgroundWork"];
1419 readonly now: DateTime.Utc;
1420 }) =>
1421 Effect.gen(function* () {
1422 const eventSink = yield* EventSink.EventSinkV2;
1423 const taskId = NodeId.make(`node:${input.name}`);
1424 const childThreadId = ThreadId.make(`thread:${input.name}`);
1425 const childRunId = RunId.make(`run:${input.name}:1`);
1426 yield* eventSink.write({
1427 commandId: CommandId.make(`command:seed-child:${input.name}`),
1428 events: [
1429 {
1430 id: EventId.make(`event:${input.name}:thread`),
1431 type: "thread.created",
1432 threadId: childThreadId,
1433 occurredAt: input.now,
1434 payload: {
1435 createdBy: "agent",
1436 creationSource: "server",
1437 id: childThreadId,
1438 projectId: input.projectId,
1439 title: input.name,
1440 providerInstanceId: modelSelection.instanceId,
1441 modelSelection,
1442 runtimeMode: "full-access",
1443 interactionMode: "default",
1444 branch: null,
1445 worktreePath: null,
1446 activeProviderThreadId: null,
1447 lineage: {
1448 parentThreadId: input.parentThreadId,
1449 relationshipToParent: "subagent",
1450 rootThreadId: input.parentThreadId,
1451 },
1452 forkedFrom: { type: "node", nodeId: taskId },
1453 createdAt: input.now,
1454 updatedAt: input.now,
1455 archivedAt: null,
1456 settledOverride: null,
1457 settledAt: null,
1458 lastVisitedAt: null,
1459 deletedAt: null,
1460 },
1461 },
1462 {
1463 id: EventId.make(`event:${input.name}:task`),
1464 type: "subagent.updated",
1465 threadId: input.parentThreadId,
1466 runId: input.parentRunId,
1467 nodeId: taskId,
1468 driver,
1469 providerInstanceId: modelSelection.instanceId,
1470 occurredAt: input.now,
1471 payload: {
1472 id: taskId,
1473 threadId: input.parentThreadId,
1474 runId: input.parentRunId,
1475 parentNodeId: input.rootNodeId,
1476 origin: "app_owned",
1477 createdBy: "agent",
1478 driver,
1479 providerInstanceId: modelSelection.instanceId,
1480 providerThreadId: null,
1481 childThreadId,
1482 nativeTaskRef: null,
1483 prompt: `Run ${input.name}.`,
1484 title: null,
1485 model: null,
1486 completionWake: input.completionWake,
1487 status: "running",
1488 result: null,
1489 startedAt: input.now,
1490 completedAt: null,
1491 updatedAt: input.now,
1492 },
1493 },
1494 ],
1495 });
1496 const childRun = runEvent({
1497 threadId: childThreadId,
1498 runId: childRunId,
1499 ordinal: 1,
1500 status: input.runStatus ?? "cancelled",
1501 now: input.now,
1502 });
1503 yield* eventSink.writeWithEffects({
1504 commandId: reconcileCommandId(input.name),
1505 events: [
1506 input.lostWork === undefined
1507 ? childRun
1508 : {
1509 ...childRun,
1510 payload: { ...childRun.payload, restartCancelledBackgroundWork: input.lostWork },
1511 },
1512 ],
1513 effects: input.continuationPending
1514 ? [
1515 {
1516 id: `effect:restart-continuation:${childRunId}`,
1517 commandId: reconcileCommandId(input.name),
1518 threadId: childThreadId,
1519 request: { type: "provider-runtime.continue", sourceRunId: childRunId },
1520 },
1521 ]
1522 : [],
1523 });
1524 return { taskId, childThreadId, childRunId };
1525 });
1526
1527 // Parent spawned two always-wake tasks: the incoming one's completion wake is queued behind a busy
1528 // run while the sibling's child run is still going.
1529 const queueCompletionWakeBesideRunningSibling = (key: string) =>
1530 Effect.gen(function* () {
1531 const orchestrator = yield* Orchestrator.OrchestratorV2;
1532 const sink = yield* EventSink.EventSinkV2;
1533 const now = yield* DateTime.now;
1534 const threadId = ThreadId.make(`${key}-parent`);
1535 const originRunId = RunId.make(`${key}-origin`);
1536 const originRootId = NodeId.make(`${key}-origin-root`);
1537 const incomingTaskId = NodeId.make(`${key}-incoming`);
1538 yield* seedParentWithTerminalTask({
1539 threadId,
1540 projectId: ProjectId.make(`${key}-project`),
1541 runId: originRunId,
1542 rootNodeId: originRootId,
1543 taskId: incomingTaskId,
1544 deliveryState: "claimed",
1545 deliveryTaskIds: [incomingTaskId],
1546 completionWake: "always",
1547 now,
1548 });
1549 yield* sink.write({
1550 events: [
1551 {
1552 id: EventId.make(`event:${key}-origin-root`),
1553 type: "node.updated",
1554 threadId,
1555 runId: originRunId,
1556 nodeId: originRootId,
1557 occurredAt: now,
1558 payload: {
1559 id: originRootId,
1560 threadId,
1561 runId: originRunId,
1562 parentNodeId: null,
1563 rootNodeId: originRootId,
1564 kind: "root_turn",
1565 status: "running",
1566 countsForRun: true,
1567 providerThreadId: null,
1568 providerTurnId: null,
1569 nativeItemRef: null,
1570 runtimeRequestId: null,
1571 checkpointScopeId: null,
1572 startedAt: now,
1573 completedAt: null,
1574 },
1575 },
1576 ],
1577 });
1578 yield* orchestrator.dispatch({
1579 type: "delegated_task.request",
1580 createdBy: "agent",
1581 creationSource: "mcp",
1582 commandId: CommandId.make(`${key}-sibling`),
1583 parentThreadId: threadId,
1584 parentRunId: originRunId,
1585 parentNodeId: originRootId,
1586 task: "Keep working",
1587 modelSelection,
1588 runtimeMode: "full-access",
1589 interactionMode: "default",
1590 completionWake: "always",
1591 });
1592 const seeded = yield* orchestrator.getThreadProjection(threadId);
1593 const origin = seeded.runs.find((run) => run.id === originRunId)!;
1594 const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
1595 // The coordinator finished its spawning turn but is busy with another run, so the wake queues.
1596 yield* sink.write({
1597 events: [
1598 {
1599 ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
1600 payload: { ...origin, status: "completed", completedAt: now },
1601 },
1602 runEvent({
1603 threadId,
1604 runId: RunId.make(`${key}-busy`),
1605 ordinal: 2,
1606 status: "running",
1607 now,
1608 }),
1609 ],
1610 });
1611 const delivery = origin.delegatedCompletion!.delivery!;
1612 yield* orchestrator.dispatch({
1613 type: "message.dispatch",
1614 createdBy: "agent",
1615 creationSource: "server",
1616 commandId: CommandId.make(`${key}-wake`),
1617 threadId,
1618 messageId: delivery.messageId,
1619 text: "Delegated task completed",
1620 attachments: [],
1621 modelSelection,
1622 dispatchMode: { type: "queue_after_active" },
1623 delegatedCompletion: {
1624 parentRunId: originRunId,
1625 generation: delivery.generation,
1626 taskIds: delivery.taskIds,
1627 },
1628 });
1629 const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1630 (run) => run.userMessageId === delivery.messageId,
1631 )!;
1632 assert.equal(wake.status, "queued");
1633 const completeSibling = Effect.gen(function* () {
1634 const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
1635 const afterSequence = yield* sink.latestSequence();
1636 yield* sink.write({
1637 events: [
1638 {
1639 ...runEvent({
1640 threadId: sibling.childThreadId!,
1641 runId: childRun.id,
1642 ordinal: childRun.ordinal,
1643 status: "completed",
1644 now,
1645 }),
1646 payload: { ...childRun, status: "completed", completedAt: now },
1647 },
1648 ],
1649 });
1650 return afterSequence;
1651 });
1652 return {
1653 orchestrator,
1654 sink,
1655 now,
1656 threadId,
1657 originRunId,
1658 incomingTaskId,
1659 sibling,
1660 delivery,
1661 wake,
1662 completeSibling,
1663 };
1664 });
1665
1666 it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1667 it.effect.each([
1668 { name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false },
1669 { name: "Stop disposes", holdQueue: true, siblingFinishesFirst: false },
1670 { name: "an interrupt reserves", holdQueue: false, siblingFinishesFirst: true },
1671 ])("$name a completion wake's other siblings", ({ holdQueue, siblingFinishesFirst }) =>
1672 Effect.gen(function* () {
1673 const orchestrator = yield* Orchestrator.OrchestratorV2;
1674 const sink = yield* EventSink.EventSinkV2;
1675 const now = yield* DateTime.now;
1676 const key = holdQueue
1677 ? "sibling-stop"
1678 : siblingFinishesFirst
1679 ? "sibling-finished-interrupt"
1680 : "sibling-interrupt";
1681 const threadId = ThreadId.make(`${key}-parent`);
1682 const originRunId = RunId.make(`${key}-origin`);
1683 const incomingTaskId = NodeId.make(`${key}-incoming`);
1684 yield* seedParentWithTerminalTask({
1685 threadId,
1686 projectId: ProjectId.make(`${key}-project`),
1687 runId: originRunId,
1688 rootNodeId: NodeId.make(`${key}-origin-root`),
1689 taskId: incomingTaskId,
1690 deliveryState: "claimed",
1691 deliveryTaskIds: [incomingTaskId],
1692 completionWake: "always",
1693 now,
1694 });
1695 const originRootId = NodeId.make(`${key}-origin-root`);
1696 yield* sink.write({
1697 events: [
1698 {
1699 id: EventId.make(`event:${key}-origin-root`),
1700 type: "node.updated",
1701 threadId,
1702 runId: originRunId,
1703 nodeId: originRootId,
1704 occurredAt: now,
1705 payload: {
1706 id: originRootId,
1707 threadId,
1708 runId: originRunId,
1709 parentNodeId: null,
1710 rootNodeId: originRootId,
1711 kind: "root_turn",
1712 status: "running",
1713 countsForRun: true,
1714 providerThreadId: null,
1715 providerTurnId: null,
1716 nativeItemRef: null,
1717 runtimeRequestId: null,
1718 checkpointScopeId: null,
1719 startedAt: now,
1720 completedAt: null,
1721 },
1722 },
1723 ],
1724 });
1725 yield* orchestrator.dispatch({
1726 type: "delegated_task.request",
1727 createdBy: "agent",
1728 creationSource: "mcp",
1729 commandId: CommandId.make(`${key}-sibling`),
1730 parentThreadId: threadId,
1731 parentRunId: originRunId,
1732 parentNodeId: originRootId,
1733 task: "Keep working",
1734 modelSelection,
1735 runtimeMode: "full-access",
1736 interactionMode: "default",
1737 completionWake: "always",
1738 });
1739 const seeded = yield* orchestrator.getThreadProjection(threadId);
1740 const origin = seeded.runs.find((run) => run.id === originRunId)!;
1741 const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
1742 yield* sink.write({
1743 events: [
1744 {
1745 ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
1746 payload: { ...origin, status: "completed", completedAt: now },
1747 },
1748 ],
1749 });
1750 const delivery = origin.delegatedCompletion!.delivery!;
1751 yield* orchestrator.dispatch({
1752 type: "message.dispatch",
1753 createdBy: "agent",
1754 creationSource: "server",
1755 commandId: CommandId.make(`${key}-wake`),
1756 threadId,
1757 messageId: delivery.messageId,
1758 text: "Delegated task completed",
1759 attachments: [],
1760 modelSelection,
1761 dispatchMode: { type: "queue_after_active" },
1762 delegatedCompletion: {
1763 parentRunId: originRunId,
1764 generation: delivery.generation,
1765 taskIds: delivery.taskIds,
1766 },
1767 });
1768 yield* orchestrator.resumeQueuedRuns;
1769 const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1770 (run) => run.userMessageId === delivery.messageId,
1771 )!;
1772 const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
1773 const completeSibling = Effect.gen(function* () {
1774 const afterSequence = yield* sink.latestSequence();
1775 yield* sink.write({
1776 events: [
1777 {
1778 ...runEvent({
1779 threadId: sibling.childThreadId!,
1780 runId: childRun.id,
1781 ordinal: childRun.ordinal,
1782 status: "completed",
1783 now,
1784 }),
1785 payload: { ...childRun, status: "completed", completedAt: now },
1786 },
1787 ],
1788 });
1789 return yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1790 Stream.filter(
1791 (stored) =>
1792 stored.event.type === "subagent.updated" &&
1793 stored.event.payload.id === sibling.id &&
1794 stored.event.payload.status === "completed",
1795 ),
1796 Stream.take(1),
1797 Stream.runDrain,
1798 Effect.as(afterSequence),
1799 );
1800 });
1801 if (siblingFinishesFirst) {
1802 yield* completeSibling;
1803 assert.equal(
1804 (yield* orchestrator.getThreadProjection(threadId)).subagents.find(
1805 (task) => task.id === sibling.id,
1806 )?.completionDelivery?.state,
1807 "pending",
1808 );
1809 }
1810 const beforeInterrupt = yield* sink.latestSequence();
1811 const interrupted = yield* orchestrator.dispatch({
1812 type: "run.interrupt",
1813 commandId: CommandId.make(`${key}-interrupt`),
1814 threadId,
1815 runId: wake.id,
1816 ...(holdQueue ? { holdQueue } : {}),
1817 });
1818 // The wake's terminal may claim the sibling right after commit, so read what the interrupt
1819 // itself wrote rather than a later projection.
1820 const interruptDelivery = new Map(
1821 interrupted.storedEvents.flatMap(({ event }) =>
1822 event.type === "subagent.updated"
1823 ? [[event.payload.id, event.payload.completionDelivery?.state] as const]
1824 : [],
1825 ),
1826 );
1827 const interruptCohort = interrupted.storedEvents.flatMap(({ event }) =>
1828 event.type === "run.updated" && event.payload.id === originRunId
1829 ? [event.payload.delegatedCompletion]
1830 : [],
1831 );
1832 assert.equal(interruptDelivery.get(incomingTaskId), "disposed");
1833 assert.equal(interruptCohort.at(-1)?.disposition, holdQueue ? "stopped" : "open");
1834 if (holdQueue) {
1835 assert.equal(interruptDelivery.get(sibling.id), "disposed");
1836 return;
1837 }
1838 assert.isFalse(interruptDelivery.has(sibling.id));
1839
1840 const afterSequence = siblingFinishesFirst ? beforeInterrupt : yield* completeSibling;
1841 // The wake's terminal, or the sibling's own completion, reserves the successor.
1842 yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1843 Stream.filter(
1844 (stored) =>
1845 stored.event.type === "subagent.updated" &&
1846 stored.event.payload.id === sibling.id &&
1847 stored.event.payload.completionDelivery?.state === "claimed",
1848 ),
1849 Stream.take(1),
1850 Stream.runDrain,
1851 );
1852 const woken = yield* orchestrator.getThreadProjection(threadId);
1853 const successor = woken.runs.find((run) => run.id === originRunId)?.delegatedCompletion
1854 ?.delivery;
1855 assert.equal(successor?.generation, delivery.generation + 1);
1856 assert.deepEqual(successor?.taskIds, [sibling.id]);
1857 assert.notEqual(successor?.messageId, delivery.messageId);
1858
1859 // ProviderContinuationService consumes the claim's offer and dispatches the successor.
1860 yield* sink.stream({ afterSequence, eventType: "message.updated" }).pipe(
1861 Stream.filter(
1862 (stored) =>
1863 stored.event.type === "message.updated" &&
1864 stored.event.payload.id === successor!.messageId,
1865 ),
1866 Stream.take(1),
1867 Stream.runDrain,
1868 );
1869 // Repeated recovery re-offers every open delivery; none may wake the parent twice.
1870 yield* orchestrator.resumeQueuedRuns;
1871 yield* orchestrator.recoverDelegatedTasks;
1872 yield* orchestrator.recoverDelegatedTasks;
1873 yield* orchestrator.resumeQueuedRuns;
1874 const delivered = yield* orchestrator.getThreadProjection(threadId);
1875 const cohort = delivered.runs.find((run) => run.id === originRunId)?.delegatedCompletion;
1876 assert.equal(cohort?.disposition, "open");
1877 assert.equal(cohort?.nextGeneration, delivery.generation + 2);
1878 assert.deepEqual(cohort?.delivery, successor);
1879 assert.deepEqual(
1880 delivered.messages
1881 .filter((message) => message.delegatedCompletion?.parentRunId === originRunId)
1882 .map((message) => [message.id, message.delegatedCompletion]),
1883 [
1884 [
1885 delivery.messageId,
1886 {
1887 parentRunId: originRunId,
1888 generation: delivery.generation,
1889 taskIds: [incomingTaskId],
1890 },
1891 ],
1892 [
1893 successor!.messageId,
1894 { parentRunId: originRunId, generation: successor!.generation, taskIds: [sibling.id] },
1895 ],
1896 ],
1897 );
1898 const successorRuns = delivered.runs.filter(
1899 (run) => run.userMessageId === successor!.messageId,
1900 );
1901 assert.equal(successorRuns.length, 1);
1902 assert.equal(successorRuns[0]?.status, "starting");
1903 assert.equal(
1904 delivered.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1905 "claimed",
1906 );
1907 }),
1908 );
1909
1910 it.effect("cancelling a queued completion wake keeps its other siblings", () =>
1911 Effect.gen(function* () {
1912 const {
1913 orchestrator,
1914 sink,
1915 threadId,
1916 originRunId,
1917 incomingTaskId,
1918 sibling,
1919 delivery,
1920 wake,
1921 completeSibling,
1922 } = yield* queueCompletionWakeBesideRunningSibling("sibling-queued-cancel");
1923 const cancelled = yield* orchestrator.dispatch({
1924 type: "queued-run.cancel",
1925 commandId: CommandId.make("sibling-queued-cancel-cancel"),
1926 threadId,
1927 runId: wake.id,
1928 });
1929 const cancelDelivery = new Map(
1930 cancelled.storedEvents.flatMap(({ event }) =>
1931 event.type === "subagent.updated"
1932 ? [[event.payload.id, event.payload.completionDelivery?.state] as const]
1933 : [],
1934 ),
1935 );
1936 assert.equal(cancelDelivery.get(incomingTaskId), "disposed");
1937 assert.notEqual(
1938 cancelDelivery.get(sibling.id),
1939 "disposed",
1940 "cancelling the wake must not dispose a still-running sibling it did not carry",
1941 );
1942 const afterCancel = yield* orchestrator.getThreadProjection(threadId);
1943 assert.equal(
1944 afterCancel.runs.find((run) => run.id === originRunId)?.delegatedCompletion?.disposition,
1945 "open",
1946 );
1947
1948 const afterSequence = yield* completeSibling;
1949 yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1950 Stream.filter(
1951 (stored) =>
1952 stored.event.type === "subagent.updated" &&
1953 stored.event.payload.id === sibling.id &&
1954 stored.event.payload.completionDelivery?.state === "claimed",
1955 ),
1956 Stream.take(1),
1957 Stream.runDrain,
1958 );
1959 const successor = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1960 (run) => run.id === originRunId,
1961 )?.delegatedCompletion?.delivery;
1962 assert.deepEqual(successor?.taskIds, [sibling.id]);
1963 assert.notEqual(successor?.messageId, delivery.messageId);
1964 }),
1965 );
1966
1967 it.effect("settling with a queued completion wake disposes its running siblings", () =>
1968 Effect.gen(function* () {
1969 const { orchestrator, sink, now, threadId, originRunId, sibling, wake, completeSibling } =
1970 yield* queueCompletionWakeBesideRunningSibling("sibling-settle");
1971 const busy = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1972 (run) => run.status === "running",
1973 )!;
1974 yield* sink.write({
1975 events: [
1976 {
1977 ...runEvent({
1978 threadId,
1979 runId: busy.id,
1980 ordinal: busy.ordinal,
1981 status: "cancelled",
1982 now,
1983 }),
1984 payload: { ...busy, status: "cancelled", completedAt: now },
1985 },
1986 {
1987 ...runEvent({ threadId, runId: wake.id, ordinal: wake.ordinal, status: "queued", now }),
1988 payload: { ...wake, queueHeld: true },
1989 },
1990 ],
1991 });
1992
1993 const settled = yield* orchestrator.dispatch({
1994 type: "thread.settle",
1995 commandId: CommandId.make("sibling-settle-settle"),
1996 threadId,
1997 });
1998 assert.deepEqual(
1999 settled.storedEvents.flatMap(({ event }) =>
2000 event.type === "run.updated" && event.payload.id === wake.id
2001 ? [event.payload.status]
2002 : [],
2003 ),
2004 ["cancelled"],
2005 );
2006 const afterSettle = yield* orchestrator.getThreadProjection(threadId);
2007 assert.equal(
2008 afterSettle.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
2009 "disposed",
2010 );
2011
2012 const afterSequence = yield* completeSibling;
2013 yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
2014 Stream.filter(
2015 (stored) =>
2016 stored.event.type === "subagent.updated" &&
2017 stored.event.payload.id === sibling.id &&
2018 stored.event.payload.status === "completed",
2019 ),
2020 Stream.take(1),
2021 Stream.runDrain,
2022 );
2023 const afterLateResult = yield* orchestrator.getThreadProjection(threadId);
2024 assert.equal(
2025 afterLateResult.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
2026 "disposed",
2027 );
2028 assert.deepEqual(
2029 afterLateResult.runs.map((run) => run.id),
2030 afterSettle.runs.map((run) => run.id),
2031 );
2032 assert.equal(
2033 afterLateResult.runs.find((run) => run.id === originRunId)?.delegatedCompletion
2034 ?.disposition,
2035 "disposed",
2036 );
2037 assert.equal(afterLateResult.thread.settledOverride, "settled");
2038 }),
2039 );
2040 });
2041
2042 it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
2043 "subagent follow-ups",
2044 (it) => {
2045 it.effect("a late unobserved read wakes a pending result without restarting", () =>
2046 Effect.gen(function* () {
2047 const orchestrator = yield* Orchestrator.OrchestratorV2;
2048 const sink = yield* EventSink.EventSinkV2;
2049 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2050 const now = yield* DateTime.now;
2051 const threadId = ThreadId.make("thread:late-read-live");
2052 const runId = RunId.make("run:late-read-live");
2053 const taskId = NodeId.make("task:late-read-live");
2054 yield* seedParentWithTerminalTask({
2055 threadId,
2056 projectId: ProjectId.make("project:late-read-live"),
2057 runId,
2058 rootNodeId: NodeId.make("root:late-read-live"),
2059 taskId,
2060 deliveryState: "pending",
2061 now,
2062 });
2063 const acknowledge = (key: string) =>
2064 orchestrator.dispatch({
2065 type: "delegated_task.completion-delivery.acknowledge",
2066 commandId: CommandId.make(`command:late-read-live:${key}`),
2067 parentThreadId: threadId,
2068 taskId,
2069 observedByRunId: null,
2070 });
2071 const before = yield* orchestrator.getThreadProjection(threadId);
2072 const liveRead = yield* acknowledge("live");
2073 assert.lengthOf(liveRead.storedEvents, 0);
2074 const parent = before.runs.find((run) => run.id === runId)!;
2075 yield* sink.write({
2076 events: [
2077 {
2078 id: EventId.make("event:late-read-live:parent-ended"),
2079 type: "run.updated",
2080 threadId,
2081 runId,
2082 occurredAt: now,
2083 payload: { ...parent, status: "completed", completedAt: now },
2084 },
2085 ],
2086 });
2087 yield* acknowledge("ended");
2088 const wake = yield* requests.take;
2089 assert.equal(wake.threadId, threadId);
2090 assert.equal(wake.delegatedCompletion?.parentRunId, runId);
2091 const after = yield* orchestrator.getThreadProjection(threadId);
2092 assert.equal(
2093 after.subagents.find((task) => task.id === taskId)?.completionDelivery?.state,
2094 "claimed",
2095 );
2096 assert.deepEqual(
2097 after.runs.find((run) => run.id === runId)?.delegatedCompletion?.delivery?.taskIds,
2098 [taskId],
2099 );
2100 const repeated = yield* acknowledge("repeated");
2101 assert.lengthOf(repeated.storedEvents, 0);
2102 assert.deepEqual(yield* orchestrator.getThreadProjection(threadId), after);
2103 yield* orchestrator.dispatch({
2104 type: "delegated_task.completion-delivery.dispose",
2105 commandId: CommandId.make("command:late-read-live:dispose"),
2106 parentThreadId: threadId,
2107 taskId,
2108 });
2109 }),
2110 );
2111
2112 it.effect("recovers a reserved delivery without loading conversation history", () =>
2113 Effect.gen(function* () {
2114 const orchestrator = yield* Orchestrator.OrchestratorV2;
2115 const sink = yield* EventSink.EventSinkV2;
2116 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2117 const now = yield* DateTime.now;
2118 const threadId = ThreadId.make("thread:bounded-delivery-recovery");
2119 const runId = RunId.make("run:bounded-delivery-recovery");
2120 const taskId = NodeId.make("task:bounded-delivery-recovery");
2121 yield* seedParentWithTerminalTask({
2122 threadId,
2123 runId,
2124 taskId,
2125 projectId: ProjectId.make("project:bounded-delivery-recovery"),
2126 rootNodeId: NodeId.make("node:bounded-delivery-recovery"),
2127 deliveryState: "claimed",
2128 deliveryTaskIds: [taskId],
2129 completionWake: "always",
2130 now,
2131 });
2132 for (const suffix of ["first", "second"]) {
2133 yield* sink.write({
2134 events: [
2135 {
2136 id: EventId.make(`event:bounded-delivery-history:${suffix}`),
2137 type: "message.updated",
2138 threadId,
2139 occurredAt: now,
2140 payload: {
2141 id: MessageId.make(`message:bounded-delivery-history:${suffix}`),
2142 threadId,
2143 runId: null,
2144 nodeId: null,
2145 createdBy: "user",
2146 creationSource: "web",
2147 role: "user",
2148 text: "Unrelated historical input",
2149 attachments: [],
2150 streaming: false,
2151 createdAt: now,
2152 updatedAt: now,
2153 },
2154 },
2155 ],
2156 });
2157 }
2158 const statements: string[] = [];
2159 const tracer = Tracer.make({
2160 span(options) {
2161 const span = new Tracer.NativeSpan(options);
2162 const end = span.end.bind(span);
2163 span.end = (time, exit) => {
2164 end(time, exit);
2165 const query = span.attributes.get("db.query.text");
2166 if (typeof query === "string") statements.push(query);
2167 };
2168 return span;
2169 },
2170 });
2171 yield* orchestrator.recoverDelegatedTasks.pipe(Effect.withTracer(tracer));
2172 assert.isFalse(statements.some((query) => query.includes("MIN(ordinal)")));
2173 const messageReads = statements.filter((query) =>
2174 query.includes("FROM orchestration_v2_projection_messages"),
2175 );
2176 assert.isNotEmpty(messageReads);
2177 assert.isTrue(messageReads.every((query) => query.includes("message_id IN")));
2178 const wake = yield* requests.take;
2179 assert.equal(wake.delegatedCompletion?.parentRunId, runId);
2180 }),
2181 );
2182 it.effect("keeps a checkpoint completion committed between finalize's read and write", () =>
2183 Effect.gen(function* () {
2184 const orchestrator = yield* Orchestrator.OrchestratorV2;
2185 const sink = yield* EventSink.EventSinkV2;
2186 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2187 const now = yield* DateTime.now;
2188 const threadId = ThreadId.make("thread:completion-capture-race");
2189 const projectId = ProjectId.make("project:completion-capture-race");
2190 const runId = RunId.make("run:completion-capture-race");
2191 const rootNodeId = NodeId.make("node:completion-capture-race");
2192 const checkpointId = CheckpointId.make("checkpoint:completion-capture-race");
2193 yield* seedParentWithTerminalTask({
2194 threadId,
2195 projectId,
2196 runId,
2197 rootNodeId,
2198 taskId: NodeId.make("node:completion-capture-seed"),
2199 deliveryState: "acknowledged",
2200 now,
2201 });
2202 const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
2203 const waiting = { ...parent, status: "waiting" as const };
2204 yield* sink.write({
2205 events: [
2206 {
2207 id: EventId.make("event:completion-capture-waiting"),
2208 type: "run.updated",
2209 threadId,
2210 runId,
2211 occurredAt: now,
2212 payload: waiting,
2213 },
2214 ],
2215 });
2216 const child = yield* seedRestartCancelledChild({
2217 parentThreadId: threadId,
2218 projectId,
2219 parentRunId: runId,
2220 rootNodeId,
2221 name: "completion-capture-child",
2222 completionWake: "always",
2223 continuationPending: false,
2224 runStatus: "completed",
2225 now,
2226 });
2227 const write = sink.writeWithEffects;
2228 let captured = false;
2229 const committedParentRuns: Array<OrchestrationV2Run> = [];
2230 yield* Effect.acquireUseRelease(
2231 Effect.sync(() =>
2232 vi.spyOn(sink, "writeWithEffects").mockImplementation((input) =>
2233 Effect.gen(function* () {
2234 if (
2235 !captured &&
2236 input.events.some(
2237 (event) =>
2238 event.type === "run.updated" &&
2239 event.payload.id === runId &&
2240 event.payload.delegatedCompletion?.delivery != null,
2241 )
2242 ) {
2243 captured = true;
2244 // Commit capture after finalize planned its stale waiting snapshot.
2245 const { delegatedCompletion: _cohort, ...captureRun } = waiting;
2246 yield* sink.commitCommand({
2247 commandId: CommandId.make("command:completion-capture-race"),
2248 threadId,
2249 commandType: "checkpoint.capture",
2250 acceptedAt: now,
2251 effects: [],
2252 events: [
2253 {
2254 id: EventId.make("event:completion-capture-completed"),
2255 type: "run.updated",
2256 threadId,
2257 runId,
2258 occurredAt: now,
2259 payload: {
2260 ...captureRun,
2261 status: "completed",
2262 completedAt: now,
2263 checkpointId,
2264 },
2265 },
2266 ],
2267 });
2268 }
2269 const stored = yield* write(input);
2270 for (const { event } of stored) {
2271 if (event.type === "run.updated" && event.payload.id === runId) {
2272 committedParentRuns.push(event.payload);
2273 }
2274 }
2275 return stored;
2276 }),
2277 ),
2278 ),
2279 () => orchestrator.recoverDelegatedTasks,
2280 (spy) => Effect.sync(() => spy.mockRestore()),
2281 );
2282 assert.isTrue(captured);
2283 assert.equal(committedParentRuns.length, 1);
2284 const committedParentRun = committedParentRuns[0];
2285 assert.equal(committedParentRun?.status, "completed");
2286 assert.deepEqual(committedParentRun?.completedAt, now);
2287 assert.equal(committedParentRun?.checkpointId, checkpointId);
2288 assert.deepEqual(committedParentRun?.delegatedCompletion?.delivery?.taskIds, [
2289 child.taskId,
2290 ]);
2291 const finished = yield* orchestrator.getThreadProjection(threadId);
2292 const completed = finished.runs.find((run) => run.id === runId)!;
2293 assert.equal(completed.status, "completed");
2294 assert.deepEqual(completed.completedAt, now);
2295 assert.equal(completed.checkpointId, checkpointId);
2296 assert.deepEqual(completed.delegatedCompletion?.delivery?.taskIds, [child.taskId]);
2297 const wake = yield* requests.take;
2298 assert.equal(wake.delegatedCompletion?.parentRunId, runId);
2299 const delivery = completed.delegatedCompletion!.delivery!;
2300 yield* orchestrator.dispatch({
2301 type: "message.dispatch",
2302 createdBy: "agent",
2303 creationSource: "server",
2304 commandId: CommandId.make("command:completion-capture-delivery"),
2305 threadId,
2306 messageId: delivery.messageId,
2307 text: "Child finished",
2308 attachments: [],
2309 delegatedCompletion: {
2310 parentRunId: runId,
2311 generation: delivery.generation,
2312 taskIds: delivery.taskIds,
2313 },
2314 dispatchMode: { type: "queue_after_active" },
2315 });
2316 const delivered = yield* orchestrator.getThreadProjection(threadId);
2317 const deliveryRun = delivered.runs.find((run) => run.userMessageId === delivery.messageId);
2318 assert.isDefined(deliveryRun);
2319 assert.notEqual(deliveryRun?.status, "queued");
2320 }),
2321 );
2322 it.effect("pauses queued follow-up cards and resumes them", () =>
2323 Effect.gen(function* () {
2324 const orchestrator = yield* Orchestrator.OrchestratorV2;
2325 const eventSink = yield* EventSink.EventSinkV2;
2326 const now = yield* DateTime.now;
2327 const parentThreadId = ThreadId.make("thread:held-followup-parent");
2328 const projectId = ProjectId.make("project:held-followup");
2329 const parentRunId = RunId.make("run:held-followup-parent");
2330 const rootNodeId = NodeId.make("node:held-followup-parent");
2331 yield* seedParentWithTerminalTask({
2332 threadId: parentThreadId,
2333 projectId,
2334 runId: parentRunId,
2335 rootNodeId,
2336 taskId: NodeId.make("node:held-followup-seed"),
2337 deliveryState: "acknowledged",
2338 now,
2339 });
2340 const child = yield* seedRestartCancelledChild({
2341 parentThreadId,
2342 projectId,
2343 parentRunId,
2344 rootNodeId,
2345 name: "held-followup-child",
2346 completionWake: "always",
2347 continuationPending: false,
2348 runStatus: "completed",
2349 now,
2350 });
2351 yield* orchestrator.recoverDelegatedTasks;
2352 const send = (name: string) =>
2353 orchestrator.dispatch({
2354 type: "message.dispatch",
2355 createdBy: "agent",
2356 creationSource: "mcp",
2357 commandId: CommandId.make(`command:held-followup:${name}`),
2358 threadId: child.childThreadId,
2359 senderThreadId: parentThreadId,
2360 messageId: MessageId.make(`message:held-followup:${name}`),
2361 text: name,
2362 attachments: [],
2363 dispatchMode: { type: "queue_after_active" },
2364 });
2365 yield* send("active");
2366 yield* send("queued");
2367 const before = yield* orchestrator.getThreadProjection(child.childThreadId);
2368 const active = before.runs.find((run) => run.ordinal === 2)!;
2369 const queued = before.runs.find((run) => run.ordinal === 3)!;
2370 assert.equal(queued.status, "queued");
2371 const waitForStatus = (afterSequence: number, status: string) =>
2372 eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
2373 Stream.filter(
2374 (stored) =>
2375 stored.event.type === "subagent.updated" &&
2376 stored.event.payload.id === queued.delegatedTaskId &&
2377 stored.event.payload.status === status,
2378 ),
2379 Stream.take(1),
2380 Stream.runHead,
2381 );
2382 const afterSequence = yield* eventSink.latestSequence();
2383 yield* orchestrator.dispatch({
2384 type: "thread.stop",
2385 threadId: child.childThreadId,
2386 commandId: CommandId.make("command:held-followup:stop"),
2387 });
2388 yield* waitForStatus(afterSequence, "idle");
2389 const paused = yield* orchestrator.getThreadProjection(parentThreadId);
2390 const task = paused.subagents.find((row) => row.id === queued.delegatedTaskId)!;
2391 assert.equal(task.status, "idle");
2392 assert.isNull(task.result);
2393 assert.equal(paused.nodes.find((row) => row.id === task.id)?.status, "idle");
2394 assert.equal(
2395 paused.turnItems.find((row) => row.type === "subagent" && row.subagentId === task.id)
2396 ?.status,
2397 "idle",
2398 );
2399 const endedSequence = yield* eventSink.latestSequence();
2400 yield* eventSink.write({
2401 events: [
2402 {
2403 id: EventId.make("event:held-followup-active-ended"),
2404 type: "run.updated",
2405 threadId: child.childThreadId,
2406 runId: active.id,
2407 occurredAt: now,
2408 payload: { ...active, status: "interrupted", startedAt: now, completedAt: now },
2409 },
2410 ],
2411 });
2412 const activeTask = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2413 (row) => row.id === active.delegatedTaskId,
2414 )!;
2415 if (activeTask.result === null) {
2416 yield* eventSink
2417 .stream({ afterSequence: endedSequence, eventType: "subagent.updated" })
2418 .pipe(
2419 Stream.filter(
2420 (stored) =>
2421 stored.event.type === "subagent.updated" &&
2422 stored.event.payload.id === active.delegatedTaskId &&
2423 stored.event.payload.status === "interrupted",
2424 ),
2425 Stream.take(1),
2426 Stream.runHead,
2427 );
2428 }
2429 const shell = (yield* orchestrator.getShellSnapshot()).threads.find(
2430 (row) => row.id === parentThreadId,
2431 )!;
2432 assert.isFalse((shell.pendingBackgroundTasks ?? []).some((row) => row.taskId === task.id));
2433 const resumeSequence = yield* eventSink.latestSequence();
2434 yield* orchestrator.dispatch({
2435 type: "queue.resume",
2436 threadId: child.childThreadId,
2437 commandId: CommandId.make("command:held-followup:resume"),
2438 });
2439 yield* waitForStatus(resumeSequence, "running");
2440 const resumed = yield* orchestrator.getThreadProjection(parentThreadId);
2441 assert.equal(resumed.subagents.find((row) => row.id === task.id)?.status, "running");
2442 assert.isNull(resumed.subagents.find((row) => row.id === task.id)?.result);
2443
2444 const started = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
2445 (run) => run.id === queued.id,
2446 )!;
2447 const waitingSequence = yield* eventSink.latestSequence();
2448 yield* eventSink.write({
2449 events: [
2450 {
2451 id: EventId.make("event:held-followup-waiting"),
2452 type: "run.updated",
2453 threadId: child.childThreadId,
2454 runId: started.id,
2455 occurredAt: now,
2456 payload: { ...started, status: "waiting" },
2457 },
2458 ],
2459 });
2460 yield* waitForStatus(waitingSequence, "waiting");
2461 const waiting = yield* orchestrator.getThreadProjection(parentThreadId);
2462 assert.equal(waiting.subagents.find((row) => row.id === task.id)?.status, "waiting");
2463 assert.equal(waiting.nodes.find((row) => row.id === task.id)?.status, "waiting");
2464 }),
2465 );
2466
2467 it.effect("checkpoint retry releases a pending sibling after its wake offer failed", () =>
2468 Effect.gen(function* () {
2469 const orchestrator = yield* Orchestrator.OrchestratorV2;
2470 const sink = yield* EventSink.EventSinkV2;
2471 const outbox = yield* EffectOutbox.EffectOutboxV2;
2472 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2473 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2474 const now = yield* DateTime.now;
2475 const threadId = ThreadId.make("thread:checkpoint-own-delivery");
2476 const originRunId = RunId.make("run:checkpoint-own-delivery:origin");
2477 const firstId = NodeId.make("node:checkpoint-own-delivery:first");
2478 const siblingId = NodeId.make("node:checkpoint-own-delivery:sibling");
2479 yield* seedParentWithTerminalTask({
2480 threadId,
2481 projectId: ProjectId.make("project:checkpoint-own-delivery"),
2482 runId: originRunId,
2483 rootNodeId: NodeId.make("node:checkpoint-own-delivery:root"),
2484 taskId: firstId,
2485 deliveryState: "claimed",
2486 deliveryTaskIds: [firstId],
2487 parentRunStatus: "completed",
2488 now,
2489 });
2490 const original = yield* orchestrator.getThreadProjection(threadId);
2491 const first = original.subagents.find((task) => task.id === firstId)!;
2492 const delivery = original.runs.find((run) => run.id === originRunId)!.delegatedCompletion!
2493 .delivery!;
2494 yield* orchestrator.dispatch({
2495 type: "message.dispatch",
2496 createdBy: "agent",
2497 creationSource: "server",
2498 commandId: reconcileCommandId("checkpoint-own-delivery-wake"),
2499 threadId,
2500 messageId: delivery.messageId,
2501 text: "First result",
2502 attachments: [],
2503 dispatchMode: { type: "queue_after_active" },
2504 delegatedCompletion: {
2505 parentRunId: originRunId,
2506 generation: delivery.generation,
2507 taskIds: delivery.taskIds,
2508 },
2509 });
2510 const started = yield* orchestrator.getThreadProjection(threadId);
2511 const run = started.runs.find(
2512 (candidate) => candidate.userMessageId === delivery.messageId,
2513 )!;
2514 const scopeId = started.nodes.find(
2515 (node) => node.id === run.rootNodeId,
2516 )!.checkpointScopeId!;
2517 yield* outbox.cancelUnsettled({
2518 threadId,
2519 effectTypes: ["provider-turn.start"],
2520 reason: "The delivery turn already completed.",
2521 });
2522 const commandId = reconcileCommandId("checkpoint-own-delivery-finished");
2523 yield* sink.writeWithEffects({
2524 commandId,
2525 events: [
2526 {
2527 id: EventId.make("event:checkpoint-own-delivery:sibling"),
2528 type: "subagent.updated",
2529 threadId,
2530 runId: originRunId,
2531 nodeId: siblingId,
2532 occurredAt: now,
2533 payload: {
2534 ...first,
2535 id: siblingId,
2536 completionDelivery: { state: "pending", observedByRunId: null },
2537 },
2538 },
2539 {
2540 id: EventId.make("event:checkpoint-own-delivery:finished"),
2541 type: "run.updated",
2542 threadId,
2543 runId: run.id,
2544 occurredAt: now,
2545 payload: {
2546 ...run,
2547 status: "completed",
2548 startedAt: now,
2549 completedAt: now,
2550 checkpointId: CheckpointId.make("checkpoint:own-delivery"),
2551 },
2552 },
2553 ],
2554 effects: [
2555 {
2556 id: "effect:checkpoint-own-delivery",
2557 commandId,
2558 threadId,
2559 request: { type: "checkpoint.capture", runId: run.id, scopeId },
2560 },
2561 ],
2562 });
2563 using offer = vi
2564 .spyOn(requests, "offer")
2565 .mockImplementationOnce(() => Effect.die("successor wake offer failed"));
2566 const firstAttempt = yield* Effect.exit(
2567 orchestrator.recoverDelegatedTask(threadId, { completedRunId: run.id }),
2568 );
2569 assert.isTrue(Exit.isFailure(firstAttempt));
2570 const reserved = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
2571 (candidate) => candidate.id === originRunId,
2572 )!.delegatedCompletion!.delivery!;
2573 assert.deepEqual(reserved.taskIds, [siblingId]);
2574 yield* worker.drain();
2575 assert.lengthOf(offer.mock.calls, 2);
2576 const completed = yield* orchestrator.getThreadProjection(threadId);
2577 assert.equal(
2578 completed.subagents.find((task) => task.id === firstId)?.completionDelivery?.state,
2579 "delivered",
2580 );
2581 assert.equal(
2582 completed.subagents.find((task) => task.id === siblingId)?.completionDelivery?.state,
2583 "claimed",
2584 );
2585 assert.deepEqual(
2586 completed.runs.find((candidate) => candidate.id === originRunId)?.delegatedCompletion
2587 ?.delivery?.taskIds,
2588 [siblingId],
2589 );
2590 let wake = yield* requests.take;
2591 while (wake.threadId !== threadId) wake = yield* requests.take;
2592 assert.equal(wake.delegatedCompletion?.generation, delivery.generation + 1);
2593 }),
2594 );
2595
2596 it.effect.each(["live", "checkpoint-retry", "checkpoint-offer-retry"] as const)(
2597 "delivers the run a background command's wake starts after the original result (%s)",
2598 (delivery) =>
2599 Effect.gen(function* () {
2600 const orchestrator = yield* Orchestrator.OrchestratorV2;
2601 const eventSink = yield* EventSink.EventSinkV2;
2602 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2603 const now = yield* DateTime.now;
2604 const parentThreadId = ThreadId.make(`thread:background-wake-parent-${delivery}`);
2605 const projectId = ProjectId.make(`project:background-wake-${delivery}`);
2606 const parentRunId = RunId.make(`run:background-wake-parent-${delivery}`);
2607 const rootNodeId = NodeId.make(`node:background-wake-parent-${delivery}`);
2608 yield* seedParentWithTerminalTask({
2609 threadId: parentThreadId,
2610 projectId,
2611 runId: parentRunId,
2612 rootNodeId,
2613 taskId: NodeId.make(`node:background-wake-seed-${delivery}`),
2614 deliveryState: "acknowledged",
2615 now,
2616 });
2617 const child = yield* seedRestartCancelledChild({
2618 parentThreadId,
2619 projectId,
2620 parentRunId,
2621 rootNodeId,
2622 name: `background-wake-child-${delivery}`,
2623 completionWake: "always",
2624 continuationPending: false,
2625 runStatus: "completed",
2626 now,
2627 });
2628 yield* orchestrator.recoverDelegatedTasks;
2629 let firstWake = yield* requests.take;
2630 while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
2631 yield* orchestrator.dispatch({
2632 type: "delegated_task.completion-delivery.acknowledge",
2633 commandId: CommandId.make(`command:background-wake-ack-${delivery}`),
2634 parentThreadId,
2635 taskId: child.taskId,
2636 observedByRunId: parentRunId,
2637 });
2638 const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2639 (run) => run.id === parentRunId,
2640 )!;
2641 yield* eventSink.write({
2642 commandId: reconcileCommandId(`background-wake-parent-finished-${delivery}`),
2643 events: [
2644 {
2645 id: EventId.make(`event:background-wake-parent-finished-${delivery}`),
2646 type: "run.updated",
2647 threadId: parentThreadId,
2648 runId: parentRunId,
2649 occurredAt: now,
2650 payload: { ...parentRun, status: "completed", completedAt: now },
2651 },
2652 ],
2653 });
2654
2655 // The shape ProviderContinuationService dispatches when the provider
2656 // reports a background command (wire kind `background_command`) ended.
2657 const wakeMessageId = MessageId.make(`message:background-wake-${delivery}`);
2658 yield* orchestrator.dispatch({
2659 type: "message.dispatch",
2660 createdBy: "agent",
2661 creationSource: "provider",
2662 commandId:
2663 delivery === "live"
2664 ? CommandId.make(`command:background-wake-${delivery}`)
2665 : reconcileCommandId(`background-wake-${delivery}`),
2666 threadId: child.childThreadId,
2667 messageId: wakeMessageId,
2668 text: "Background activity updated",
2669 // A monitor that ends is finished work too; only its per-line updates are progress.
2670 notification:
2671 delivery === "live"
2672 ? {
2673 source: { kind: "monitor" },
2674 outcome: "completed",
2675 summary: 'Monitor "sleep 60" finished',
2676 }
2677 : {
2678 source: { kind: "command" },
2679 outcome: "completed",
2680 summary: 'Command "sleep 60" finished',
2681 },
2682 attachments: [],
2683 dispatchMode: { type: "queue_after_active" },
2684 });
2685 const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId);
2686 const run = childProjection.runs.find((row) => row.userMessageId === wakeMessageId)!;
2687 const scopeId = childProjection.nodes.find(
2688 (node) => node.id === run.rootNodeId,
2689 )!.checkpointScopeId!;
2690 const commandId =
2691 delivery === "live"
2692 ? CommandId.make(`command:background-wake-finished-${delivery}`)
2693 : reconcileCommandId(`background-wake-finished-${delivery}`);
2694 if (delivery !== "live") {
2695 const outbox = yield* EffectOutbox.EffectOutboxV2;
2696 yield* outbox.cancelUnsettled({
2697 threadId: child.childThreadId,
2698 effectTypes: ["provider-turn.start"],
2699 reason: "The turn already finished before its checkpoint retry.",
2700 });
2701 }
2702 const afterSequence = yield* eventSink.latestSequence();
2703 assert.isDefined(run.delegatedTaskId);
2704 yield* eventSink.writeWithEffects({
2705 commandId,
2706 effects:
2707 delivery === "live"
2708 ? []
2709 : [
2710 {
2711 id: `effect:background-wake-checkpoint-retry-${delivery}`,
2712 commandId,
2713 threadId: child.childThreadId,
2714 request: { type: "checkpoint.capture", runId: run.id, scopeId },
2715 },
2716 ],
2717 events: [
2718 {
2719 id: EventId.make(`event:background-wake-result-${delivery}`),
2720 type: "message.updated",
2721 threadId: child.childThreadId,
2722 runId: run.id,
2723 occurredAt: now,
2724 payload: {
2725 id: MessageId.make(`message:background-wake-result-${delivery}`),
2726 threadId: child.childThreadId,
2727 runId: run.id,
2728 nodeId: null,
2729 role: "assistant",
2730 text: "The background command finished cleanly.",
2731 attachments: [],
2732 streaming: false,
2733 createdBy: "agent",
2734 creationSource: "server",
2735 createdAt: now,
2736 updatedAt: now,
2737 },
2738 },
2739 {
2740 id: EventId.make(`event:background-wake-finished-${delivery}`),
2741 type: "run.updated",
2742 threadId: child.childThreadId,
2743 runId: run.id,
2744 occurredAt: now,
2745 payload: {
2746 ...run,
2747 status: "completed",
2748 startedAt: now,
2749 completedAt: now,
2750 checkpointId:
2751 delivery === "live"
2752 ? null
2753 : CheckpointId.make(`checkpoint:background-wake-${delivery}`),
2754 },
2755 },
2756 ],
2757 });
2758 if (delivery !== "live") {
2759 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2760 if (delivery === "checkpoint-offer-retry") {
2761 using offer = vi
2762 .spyOn(requests, "offer")
2763 .mockImplementationOnce(() => Effect.die("wake offer failed"));
2764 const first = yield* Effect.exit(
2765 orchestrator.recoverDelegatedTask(child.childThreadId),
2766 );
2767 assert.isTrue(Exit.isFailure(first));
2768 const committed = (yield* orchestrator.getThreadProjection(
2769 parentThreadId,
2770 )).subagents.find((task) => task.id === run.delegatedTaskId)!;
2771 assert.equal(committed.result, "The background command finished cleanly.");
2772 yield* worker.drain();
2773 assert.lengthOf(offer.mock.calls, 2);
2774 } else {
2775 yield* worker.drain();
2776 }
2777 } else {
2778 yield* eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
2779 Stream.filter(
2780 (stored) =>
2781 stored.event.type === "subagent.updated" &&
2782 stored.event.payload.id === run.delegatedTaskId &&
2783 stored.event.payload.status === "completed",
2784 ),
2785 Stream.take(1),
2786 Stream.runDrain,
2787 );
2788 }
2789 const completed = yield* orchestrator.getThreadProjection(parentThreadId);
2790 const followup = completed.subagents.find((task) => task.id === run.delegatedTaskId)!;
2791 assert.lengthOf(
2792 completed.contextTransfers.filter(
2793 (transfer) =>
2794 transfer.type === "subagent_result" &&
2795 transfer.sourceThreadId === child.childThreadId,
2796 ),
2797 2,
2798 );
2799 assert.equal(followup.status, "completed");
2800 assert.equal(followup.result, "The background command finished cleanly.");
2801 assert.equal(followup.completionDelivery?.state, "claimed");
2802 let wake = yield* requests.take;
2803 while (
2804 wake.threadId !== parentThreadId ||
2805 wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId
2806 ) {
2807 wake = yield* requests.take;
2808 }
2809 assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId);
2810 }),
2811 );
2812
2813 // The child's follow-up run sends `t3_thread_send` to its own thread before it ends.
2814 const startSelfQueuedFollowup = (name: string) =>
2815 Effect.gen(function* () {
2816 const orchestrator = yield* Orchestrator.OrchestratorV2;
2817 const eventSink = yield* EventSink.EventSinkV2;
2818 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2819 const now = yield* DateTime.now;
2820 const parentThreadId = ThreadId.make(`thread:${name}-parent`);
2821 const projectId = ProjectId.make(`project:${name}`);
2822 const parentRunId = RunId.make(`run:${name}-parent`);
2823 const rootNodeId = NodeId.make(`node:${name}-parent`);
2824 yield* seedParentWithTerminalTask({
2825 threadId: parentThreadId,
2826 projectId,
2827 runId: parentRunId,
2828 rootNodeId,
2829 taskId: NodeId.make(`node:${name}-seed`),
2830 deliveryState: "acknowledged",
2831 now,
2832 });
2833 const child = yield* seedRestartCancelledChild({
2834 parentThreadId,
2835 projectId,
2836 parentRunId,
2837 rootNodeId,
2838 name: `${name}-child`,
2839 completionWake: "always",
2840 continuationPending: false,
2841 runStatus: "completed",
2842 now,
2843 });
2844 yield* orchestrator.recoverDelegatedTasks;
2845 let firstWake = yield* requests.take;
2846 while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
2847 yield* orchestrator.dispatch({
2848 type: "delegated_task.completion-delivery.acknowledge",
2849 commandId: CommandId.make(`command:${name}-ack`),
2850 parentThreadId,
2851 taskId: child.taskId,
2852 observedByRunId: parentRunId,
2853 });
2854 const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2855 (run) => run.id === parentRunId,
2856 )!;
2857 yield* eventSink.write({
2858 commandId: reconcileCommandId(`${name}-parent-finished`),
2859 events: [
2860 {
2861 id: EventId.make(`event:${name}-parent-finished`),
2862 type: "run.updated",
2863 threadId: parentThreadId,
2864 runId: parentRunId,
2865 occurredAt: now,
2866 payload: { ...parentRun, status: "completed", completedAt: now },
2867 },
2868 ],
2869 });
2870 const followupMessageId = MessageId.make(`message:${name}`);
2871 yield* orchestrator.dispatch({
2872 type: "message.dispatch",
2873 createdBy: "agent",
2874 creationSource: "mcp",
2875 commandId: CommandId.make(`command:${name}`),
2876 threadId: child.childThreadId,
2877 senderThreadId: parentThreadId,
2878 messageId: followupMessageId,
2879 text: "Check the same work again.",
2880 attachments: [],
2881 dispatchMode: { type: "start_immediately" },
2882 });
2883 const queuedRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
2884 (row) => row.userMessageId === followupMessageId,
2885 )!;
2886 const followupTaskId = queuedRun.delegatedTaskId;
2887 assert.isDefined(followupTaskId);
2888 const followupRun = { ...queuedRun, status: "running" as const, startedAt: now };
2889 yield* eventSink.write({
2890 commandId: reconcileCommandId(`${name}-running`),
2891 events: [
2892 {
2893 id: EventId.make(`event:${name}-running`),
2894 type: "run.updated",
2895 threadId: child.childThreadId,
2896 runId: followupRun.id,
2897 occurredAt: now,
2898 payload: followupRun,
2899 },
2900 ],
2901 });
2902 const selfQueuedMessageId = MessageId.make(`message:${name}-self-send`);
2903 yield* orchestrator.dispatch({
2904 type: "message.dispatch",
2905 createdBy: "agent",
2906 creationSource: "mcp",
2907 commandId: CommandId.make(`command:${name}-self-send`),
2908 threadId: child.childThreadId,
2909 senderThreadId: child.childThreadId,
2910 messageId: selfQueuedMessageId,
2911 text: "Finish the work after this turn.",
2912 attachments: [],
2913 dispatchMode: { type: "queue_after_active" },
2914 });
2915 const selfQueuedRun = (yield* orchestrator.getThreadProjection(
2916 child.childThreadId,
2917 )).runs.find((row) => row.userMessageId === selfQueuedMessageId)!;
2918 assert.equal(selfQueuedRun.status, "queued");
2919 const finishRun = (step: string, run: OrchestrationV2Run, text: string) =>
2920 eventSink.write({
2921 commandId: reconcileCommandId(`${name}-${step}`),
2922 events: [
2923 {
2924 id: EventId.make(`event:${name}-${step}:result`),
2925 type: "message.updated",
2926 threadId: child.childThreadId,
2927 runId: run.id,
2928 occurredAt: now,
2929 payload: {
2930 id: MessageId.make(`message:${name}-${step}:result`),
2931 threadId: child.childThreadId,
2932 runId: run.id,
2933 nodeId: null,
2934 role: "assistant",
2935 text,
2936 attachments: [],
2937 streaming: false,
2938 createdBy: "agent",
2939 creationSource: "server",
2940 createdAt: now,
2941 updatedAt: now,
2942 },
2943 },
2944 {
2945 id: EventId.make(`event:${name}-${step}:finished`),
2946 type: "run.updated",
2947 threadId: child.childThreadId,
2948 runId: run.id,
2949 occurredAt: now,
2950 payload: { ...run, status: "completed", startedAt: now, completedAt: now },
2951 },
2952 ],
2953 });
2954 const removeSelfQueuedRun = orchestrator.dispatch({
2955 type: "queued-run.cancel",
2956 commandId: CommandId.make(`command:${name}-remove-self-send`),
2957 threadId: child.childThreadId,
2958 runId: selfQueuedRun.id,
2959 });
2960 const readFollowup = Effect.gen(function* () {
2961 yield* orchestrator.recoverDelegatedTasks;
2962 return (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2963 (task) => task.id === followupTaskId,
2964 )!;
2965 });
2966 return {
2967 parentThreadId,
2968 parentRunId,
2969 childThreadId: child.childThreadId,
2970 firstWake,
2971 followupRun,
2972 selfQueuedRun,
2973 finishRun,
2974 removeSelfQueuedRun,
2975 readFollowup,
2976 };
2977 });
2978
2979 it.effect("holds a follow-up open while the child's own subagent still runs", () =>
2980 Effect.gen(function* () {
2981 const orchestrator = yield* Orchestrator.OrchestratorV2;
2982 const eventSink = yield* EventSink.EventSinkV2;
2983 const now = yield* DateTime.now;
2984 const parentThreadId = ThreadId.make("thread:followup-grandchild-parent");
2985 const projectId = ProjectId.make("project:followup-grandchild");
2986 const parentRunId = RunId.make("run:followup-grandchild-parent");
2987 const rootNodeId = NodeId.make("node:followup-grandchild-parent");
2988 yield* seedParentWithTerminalTask({
2989 threadId: parentThreadId,
2990 projectId,
2991 runId: parentRunId,
2992 rootNodeId,
2993 taskId: NodeId.make("node:followup-grandchild-seed"),
2994 deliveryState: "acknowledged",
2995 now,
2996 });
2997 const child = yield* seedRestartCancelledChild({
2998 parentThreadId,
2999 projectId,
3000 parentRunId,
3001 rootNodeId,
3002 name: "followup-grandchild-child",
3003 completionWake: "always",
3004 continuationPending: false,
3005 runStatus: "completed",
3006 now,
3007 });
3008 yield* orchestrator.recoverDelegatedTasks;
3009 assert.equal(
3010 (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3011 (task) => task.id === child.taskId,
3012 )?.status,
3013 "completed",
3014 );
3015
3016 const followupMessageId = MessageId.make("message:followup-grandchild-send");
3017 yield* orchestrator.dispatch({
3018 type: "message.dispatch",
3019 createdBy: "agent",
3020 creationSource: "mcp",
3021 commandId: CommandId.make("command:followup-grandchild-send"),
3022 threadId: child.childThreadId,
3023 senderThreadId: parentThreadId,
3024 messageId: followupMessageId,
3025 text: "Now delegate the next step.",
3026 attachments: [],
3027 dispatchMode: { type: "queue_after_active" },
3028 });
3029 const followupRun = (yield* orchestrator.getThreadProjection(
3030 child.childThreadId,
3031 )).runs.find((run) => run.userMessageId === followupMessageId)!;
3032 assert.isDefined(followupRun.delegatedTaskId);
3033 const followupTaskId = followupRun.delegatedTaskId!;
3034
3035 const grandchildId = NodeId.make("node:followup-grandchild-grandchild");
3036 const grandchild = {
3037 id: grandchildId,
3038 threadId: child.childThreadId,
3039 runId: followupRun.id,
3040 parentNodeId: NodeId.make("node:followup-grandchild-child-root"),
3041 origin: "app_owned" as const,
3042 createdBy: "agent" as const,
3043 driver,
3044 providerInstanceId: modelSelection.instanceId,
3045 providerThreadId: null,
3046 childThreadId: ThreadId.make("thread:followup-grandchild-grandchild"),
3047 nativeTaskRef: null,
3048 prompt: "Do the next step.",
3049 title: null,
3050 model: null,
3051 completionWake: "always" as const,
3052 status: "running" as const,
3053 result: null,
3054 startedAt: now,
3055 completedAt: null,
3056 updatedAt: now,
3057 };
3058 yield* eventSink.write({
3059 commandId: reconcileCommandId("followup-grandchild-turn-ended"),
3060 events: [
3061 {
3062 id: EventId.make("event:followup-grandchild-started"),
3063 type: "subagent.updated",
3064 threadId: child.childThreadId,
3065 runId: followupRun.id,
3066 nodeId: grandchildId,
3067 driver,
3068 providerInstanceId: modelSelection.instanceId,
3069 occurredAt: now,
3070 payload: grandchild,
3071 },
3072 {
3073 id: EventId.make("event:followup-grandchild-run-completed"),
3074 type: "run.updated",
3075 threadId: child.childThreadId,
3076 runId: followupRun.id,
3077 occurredAt: now,
3078 payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now },
3079 },
3080 ],
3081 });
3082 yield* orchestrator.recoverDelegatedTasks;
3083
3084 const waiting = yield* orchestrator.getThreadProjection(parentThreadId);
3085 const heldFollowup = waiting.subagents.find((task) => task.id === followupTaskId);
3086 assert.isNull(
3087 heldFollowup?.result ?? null,
3088 "the follow-up closed while the child's own subagent was still running",
3089 );
3090 assert.equal(heldFollowup?.status, "running");
3091 assert.equal(
3092 waiting.contextTransfers.filter(
3093 (transfer) =>
3094 transfer.type === "subagent_result" &&
3095 transfer.sourceThreadId === child.childThreadId,
3096 ).length,
3097 1,
3098 );
3099
3100 // Another turn in the child does not release the hold while the grandchild runs.
3101 for (const status of ["running", "completed"] as const) {
3102 yield* eventSink.write({
3103 commandId: reconcileCommandId(`followup-grandchild-user-${status}`),
3104 events: [
3105 runEvent({
3106 threadId: child.childThreadId,
3107 runId: RunId.make("run:followup-grandchild-user"),
3108 ordinal: followupRun.ordinal + 1,
3109 status,
3110 now,
3111 }),
3112 ],
3113 });
3114 yield* orchestrator.recoverDelegatedTasks;
3115 assert.isNull(
3116 (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3117 (task) => task.id === followupTaskId,
3118 )?.result ?? null,
3119 `the follow-up closed while another child run was ${status}`,
3120 );
3121 }
3122
3123 // The grandchild's completion wakes the child in a run without a delegatedTaskId.
3124 const wakeRunId = RunId.make("run:followup-grandchild-wake");
3125 const wakeRun = (status: "running" | "completed") =>
3126 runEvent({
3127 threadId: child.childThreadId,
3128 runId: wakeRunId,
3129 ordinal: followupRun.ordinal + 2,
3130 status,
3131 now,
3132 });
3133 yield* eventSink.write({
3134 commandId: reconcileCommandId("followup-grandchild-settled"),
3135 events: [
3136 {
3137 id: EventId.make("event:followup-grandchild-completed"),
3138 type: "subagent.updated",
3139 threadId: child.childThreadId,
3140 runId: followupRun.id,
3141 nodeId: grandchildId,
3142 driver,
3143 providerInstanceId: modelSelection.instanceId,
3144 occurredAt: now,
3145 payload: {
3146 ...grandchild,
3147 status: "completed",
3148 result: "grandchild finished",
3149 completionDelivery: { state: "acknowledged", observedByRunId: wakeRunId },
3150 completedAt: now,
3151 },
3152 },
3153 {
3154 id: EventId.make("event:followup-grandchild-wake-message"),
3155 type: "message.updated",
3156 threadId: child.childThreadId,
3157 runId: wakeRunId,
3158 occurredAt: now,
3159 payload: {
3160 id: MessageId.make(`message:${wakeRunId}`),
3161 threadId: child.childThreadId,
3162 runId: wakeRunId,
3163 nodeId: null,
3164 role: "user",
3165 text: "Delegated task finished.",
3166 attachments: [],
3167 streaming: false,
3168 createdBy: "agent",
3169 creationSource: "server",
3170 delegatedCompletion: {
3171 parentRunId: followupRun.id,
3172 generation: 1,
3173 taskIds: [grandchildId],
3174 },
3175 createdAt: now,
3176 updatedAt: now,
3177 },
3178 },
3179 wakeRun("running"),
3180 ],
3181 });
3182 yield* orchestrator.recoverDelegatedTasks;
3183 const waking = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3184 (task) => task.id === followupTaskId,
3185 );
3186 assert.isNull(waking?.result ?? null, "the follow-up closed before the child's wake ended");
3187
3188 yield* eventSink.write({
3189 commandId: reconcileCommandId("followup-grandchild-wake-finished"),
3190 events: [
3191 {
3192 id: EventId.make("event:followup-grandchild-wake-result"),
3193 type: "message.updated",
3194 threadId: child.childThreadId,
3195 runId: wakeRunId,
3196 occurredAt: now,
3197 payload: {
3198 id: MessageId.make("message:followup-grandchild-wake-result"),
3199 threadId: child.childThreadId,
3200 runId: wakeRunId,
3201 nodeId: null,
3202 role: "assistant",
3203 text: "The grandchild finished the next step.",
3204 attachments: [],
3205 streaming: false,
3206 createdBy: "agent",
3207 creationSource: "server",
3208 createdAt: now,
3209 updatedAt: now,
3210 },
3211 },
3212 wakeRun("completed"),
3213 ],
3214 });
3215 yield* orchestrator.recoverDelegatedTasks;
3216 const settled = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3217 (task) => task.id === followupTaskId,
3218 );
3219 assert.equal(settled?.status, "completed");
3220 assert.equal(settled?.result, "The grandchild finished the next step.");
3221 }),
3222 );
3223
3224 it.effect.each(["woken", "cancelled"] as const)(
3225 "holds a follow-up open while the child's awaited background command runs (%s)",
3226 (ending) =>
3227 Effect.gen(function* () {
3228 const orchestrator = yield* Orchestrator.OrchestratorV2;
3229 const eventSink = yield* EventSink.EventSinkV2;
3230 const now = yield* DateTime.now;
3231 const parentThreadId = ThreadId.make(`thread:followup-command-parent-${ending}`);
3232 const projectId = ProjectId.make(`project:followup-command-${ending}`);
3233 const parentRunId = RunId.make(`run:followup-command-parent-${ending}`);
3234 const rootNodeId = NodeId.make(`node:followup-command-parent-${ending}`);
3235 yield* seedParentWithTerminalTask({
3236 threadId: parentThreadId,
3237 projectId,
3238 runId: parentRunId,
3239 rootNodeId,
3240 taskId: NodeId.make(`node:followup-command-seed-${ending}`),
3241 deliveryState: "acknowledged",
3242 now,
3243 });
3244 const child = yield* seedRestartCancelledChild({
3245 parentThreadId,
3246 projectId,
3247 parentRunId,
3248 rootNodeId,
3249 name: `followup-command-child-${ending}`,
3250 completionWake: "always",
3251 continuationPending: false,
3252 runStatus: "completed",
3253 now,
3254 });
3255 yield* orchestrator.recoverDelegatedTasks;
3256
3257 const followupMessageId = MessageId.make(`message:followup-command-send-${ending}`);
3258 yield* orchestrator.dispatch({
3259 type: "message.dispatch",
3260 createdBy: "agent",
3261 creationSource: "mcp",
3262 commandId: CommandId.make(`command:followup-command-send-${ending}`),
3263 threadId: child.childThreadId,
3264 senderThreadId: parentThreadId,
3265 messageId: followupMessageId,
3266 text: "Start the check in the background, wait on it, and reply when it ends.",
3267 attachments: [],
3268 dispatchMode: { type: "queue_after_active" },
3269 });
3270 const followupRun = (yield* orchestrator.getThreadProjection(
3271 child.childThreadId,
3272 )).runs.find((run) => run.userMessageId === followupMessageId)!;
3273 const followupTaskId = followupRun.delegatedTaskId!;
3274 const parent = yield* orchestrator.getThreadProjection(parentThreadId);
3275
3276 // The child awaits a background command, replies with an interim note and ends its turn.
3277 yield* eventSink.write({
3278 commandId: reconcileCommandId(`followup-command-turn-ended-${ending}`),
3279 events: [
3280 {
3281 id: EventId.make(`event:followup-command-awaited-${ending}`),
3282 type: "provider-thread.updated",
3283 threadId: child.childThreadId,
3284 runId: followupRun.id,
3285 driver,
3286 providerInstanceId: modelSelection.instanceId,
3287 occurredAt: now,
3288 payload: {
3289 ...parent.providerThreads[0]!,
3290 id: ProviderThreadId.make(`provider-thread:followup-command-child-${ending}`),
3291 appThreadId: child.childThreadId,
3292 pendingBackgroundTasks: [
3293 { taskId: "sleep-90", kind: "command" as const, awaited: true as const },
3294 ],
3295 },
3296 },
3297 {
3298 id: EventId.make(`event:followup-command-interim-${ending}`),
3299 type: "message.updated",
3300 threadId: child.childThreadId,
3301 runId: followupRun.id,
3302 occurredAt: now,
3303 payload: {
3304 id: MessageId.make(`message:followup-command-interim-${ending}`),
3305 threadId: child.childThreadId,
3306 runId: followupRun.id,
3307 nodeId: null,
3308 role: "assistant",
3309 text: "INTERIM",
3310 attachments: [],
3311 streaming: false,
3312 createdBy: "agent",
3313 creationSource: "server",
3314 createdAt: now,
3315 updatedAt: now,
3316 },
3317 },
3318 {
3319 id: EventId.make(`event:followup-command-run-completed-${ending}`),
3320 type: "run.updated",
3321 threadId: child.childThreadId,
3322 runId: followupRun.id,
3323 occurredAt: now,
3324 payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now },
3325 },
3326 ],
3327 });
3328 yield* orchestrator.recoverDelegatedTasks;
3329
3330 const held = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3331 (task) => task.id === followupTaskId,
3332 );
3333 assert.isNull(
3334 held?.result ?? null,
3335 "the follow-up closed with its interim reply while the awaited command still ran",
3336 );
3337 assert.equal(held?.status, "running");
3338
3339 if (ending === "cancelled") {
3340 yield* orchestrator.cancelDelegatedTask({
3341 parentThreadId,
3342 taskId: followupTaskId,
3343 command: {
3344 type: "thread.stop",
3345 commandId: CommandId.make(`command:followup-command-cancel-${ending}`),
3346 threadId: child.childThreadId,
3347 stopDelegatedTasks: false,
3348 },
3349 stopDescendants: () => Effect.void,
3350 });
3351 yield* eventSink.write({
3352 commandId: reconcileCommandId(`followup-command-stopped-${ending}`),
3353 events: [
3354 {
3355 id: EventId.make(`event:followup-command-stopped-${ending}`),
3356 type: "provider-thread.updated",
3357 threadId: child.childThreadId,
3358 driver,
3359 providerInstanceId: modelSelection.instanceId,
3360 occurredAt: now,
3361 payload: {
3362 ...parent.providerThreads[0]!,
3363 id: ProviderThreadId.make(`provider-thread:followup-command-child-${ending}`),
3364 appThreadId: child.childThreadId,
3365 pendingBackgroundTasks: [],
3366 },
3367 },
3368 ],
3369 });
3370 yield* orchestrator.recoverDelegatedTasks;
3371 const stopped = (yield* orchestrator.getThreadProjection(
3372 parentThreadId,
3373 )).subagents.find((task) => task.id === followupTaskId);
3374 assert.equal(stopped?.cancelledRunId, followupRun.id);
3375 assert.equal(stopped?.status, "interrupted");
3376 return;
3377 }
3378
3379 // The finished command wakes the child; that turn continues the same follow-up.
3380 const wakeMessageId = MessageId.make(`message:followup-command-wake-${ending}`);
3381 yield* orchestrator.dispatch({
3382 type: "message.dispatch",
3383 createdBy: "agent",
3384 creationSource: "server",
3385 commandId: CommandId.make(`command:followup-command-wake-${ending}`),
3386 threadId: child.childThreadId,
3387 messageId: wakeMessageId,
3388 text: "Background activity updated",
3389 notification: {
3390 source: { kind: "command" },
3391 outcome: "completed",
3392 summary: 'Command "sleep 90" finished',
3393 },
3394 attachments: [],
3395 dispatchMode: { type: "queue_after_active" },
3396 });
3397 const wakeRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
3398 (run) => run.userMessageId === wakeMessageId,
3399 )!;
3400 assert.equal(wakeRun.delegatedTaskId, followupTaskId);
3401 yield* eventSink.write({
3402 commandId: reconcileCommandId(`followup-command-wake-ended-${ending}`),
3403 events: [
3404 {
3405 id: EventId.make(`event:followup-command-settled-${ending}`),
3406 type: "provider-thread.updated",
3407 threadId: child.childThreadId,
3408 driver,
3409 providerInstanceId: modelSelection.instanceId,
3410 occurredAt: now,
3411 payload: {
3412 ...parent.providerThreads[0]!,
3413 id: ProviderThreadId.make(`provider-thread:followup-command-child-${ending}`),
3414 appThreadId: child.childThreadId,
3415 pendingBackgroundTasks: [],
3416 },
3417 },
3418 {
3419 id: EventId.make(`event:followup-command-final-${ending}`),
3420 type: "message.updated",
3421 threadId: child.childThreadId,
3422 runId: wakeRun.id,
3423 occurredAt: now,
3424 payload: {
3425 id: MessageId.make(`message:followup-command-final-${ending}`),
3426 threadId: child.childThreadId,
3427 runId: wakeRun.id,
3428 nodeId: null,
3429 role: "assistant",
3430 text: "FINAL CHECK-OK",
3431 attachments: [],
3432 streaming: false,
3433 createdBy: "agent",
3434 creationSource: "server",
3435 createdAt: now,
3436 updatedAt: now,
3437 },
3438 },
3439 {
3440 id: EventId.make(`event:followup-command-wake-completed-${ending}`),
3441 type: "run.updated",
3442 threadId: child.childThreadId,
3443 runId: wakeRun.id,
3444 occurredAt: now,
3445 payload: { ...wakeRun, status: "completed", startedAt: now, completedAt: now },
3446 },
3447 ],
3448 });
3449 yield* orchestrator.recoverDelegatedTasks;
3450
3451 const finished = yield* orchestrator.getThreadProjection(parentThreadId);
3452 const childTasks = finished.subagents.filter(
3453 (task) => task.childThreadId === child.childThreadId,
3454 );
3455 assert.deepEqual(
3456 childTasks.map((task) => task.id).toSorted(),
3457 [child.taskId, followupTaskId].toSorted(),
3458 );
3459 const followup = childTasks.find((task) => task.id === followupTaskId);
3460 assert.equal(followup?.status, "completed");
3461 assert.equal(followup?.result, "FINAL CHECK-OK");
3462 }),
3463 );
3464
3465 it.effect("settles a follow-up whose own run started no background work", () =>
3466 Effect.gen(function* () {
3467 const orchestrator = yield* Orchestrator.OrchestratorV2;
3468 const eventSink = yield* EventSink.EventSinkV2;
3469 const now = yield* DateTime.now;
3470 const parentThreadId = ThreadId.make("thread:followup-earlier-work-parent");
3471 const projectId = ProjectId.make("project:followup-earlier-work");
3472 const parentRunId = RunId.make("run:followup-earlier-work-parent");
3473 const rootNodeId = NodeId.make("node:followup-earlier-work-parent");
3474 yield* seedParentWithTerminalTask({
3475 threadId: parentThreadId,
3476 projectId,
3477 runId: parentRunId,
3478 rootNodeId,
3479 taskId: NodeId.make("node:followup-earlier-work-seed"),
3480 deliveryState: "acknowledged",
3481 now,
3482 });
3483 const child = yield* seedRestartCancelledChild({
3484 parentThreadId,
3485 projectId,
3486 parentRunId,
3487 rootNodeId,
3488 name: "followup-earlier-work-child",
3489 completionWake: "always",
3490 continuationPending: false,
3491 runStatus: "completed",
3492 now,
3493 });
3494 yield* orchestrator.recoverDelegatedTasks;
3495 const parent = yield* orchestrator.getThreadProjection(parentThreadId);
3496 const childProviderThread = {
3497 ...parent.providerThreads[0]!,
3498 id: ProviderThreadId.make("provider-thread:followup-earlier-work-child"),
3499 appThreadId: child.childThreadId,
3500 };
3501
3502 // An earlier run of the child left an awaited build running.
3503 yield* eventSink.write({
3504 commandId: reconcileCommandId("followup-earlier-work-build"),
3505 events: [
3506 {
3507 id: EventId.make("event:followup-earlier-work-build"),
3508 type: "provider-thread.updated",
3509 threadId: child.childThreadId,
3510 runId: child.childRunId,
3511 driver,
3512 providerInstanceId: modelSelection.instanceId,
3513 occurredAt: now,
3514 payload: {
3515 ...childProviderThread,
3516 pendingBackgroundTasks: [
3517 { taskId: "build", kind: "command" as const, awaited: true as const },
3518 ],
3519 },
3520 },
3521 ],
3522 });
3523
3524 const followupMessageId = MessageId.make("message:followup-earlier-work-send");
3525 yield* orchestrator.dispatch({
3526 type: "message.dispatch",
3527 createdBy: "agent",
3528 creationSource: "mcp",
3529 commandId: CommandId.make("command:followup-earlier-work-send"),
3530 threadId: child.childThreadId,
3531 senderThreadId: parentThreadId,
3532 messageId: followupMessageId,
3533 text: "Reply with exactly PONG.",
3534 attachments: [],
3535 dispatchMode: { type: "queue_after_active" },
3536 });
3537 const followupRun = (yield* orchestrator.getThreadProjection(
3538 child.childThreadId,
3539 )).runs.find((run) => run.userMessageId === followupMessageId)!;
3540 const followupTaskId = followupRun.delegatedTaskId!;
3541
3542 // The follow-up's run answers and ends without starting any background work.
3543 yield* eventSink.write({
3544 commandId: reconcileCommandId("followup-earlier-work-answered"),
3545 events: [
3546 // The provider resends its whole roster during the follow-up's run.
3547 {
3548 id: EventId.make("event:followup-earlier-work-roster"),
3549 type: "provider-thread.updated",
3550 threadId: child.childThreadId,
3551 runId: followupRun.id,
3552 driver,
3553 providerInstanceId: modelSelection.instanceId,
3554 occurredAt: now,
3555 payload: {
3556 ...childProviderThread,
3557 pendingBackgroundTasks: [
3558 { taskId: "build", kind: "command" as const, awaited: true as const },
3559 ],
3560 },
3561 },
3562 {
3563 id: EventId.make("event:followup-earlier-work-answer"),
3564 type: "message.updated",
3565 threadId: child.childThreadId,
3566 runId: followupRun.id,
3567 occurredAt: now,
3568 payload: {
3569 id: MessageId.make("message:followup-earlier-work-answer"),
3570 threadId: child.childThreadId,
3571 runId: followupRun.id,
3572 nodeId: null,
3573 role: "assistant",
3574 text: "PONG",
3575 attachments: [],
3576 streaming: false,
3577 createdBy: "agent",
3578 creationSource: "server",
3579 createdAt: now,
3580 updatedAt: now,
3581 },
3582 },
3583 {
3584 id: EventId.make("event:followup-earlier-work-run-completed"),
3585 type: "run.updated",
3586 threadId: child.childThreadId,
3587 runId: followupRun.id,
3588 occurredAt: now,
3589 payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now },
3590 },
3591 ],
3592 });
3593 yield* orchestrator.recoverDelegatedTasks;
3594
3595 const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3596 (task) => task.id === followupTaskId,
3597 );
3598 assert.equal(followup?.status, "completed");
3599 assert.equal(followup?.result, "PONG");
3600 }),
3601 );
3602
3603 it.effect("tracks a user's follow-up but not the compaction sent ahead of it", () =>
3604 Effect.gen(function* () {
3605 const orchestrator = yield* Orchestrator.OrchestratorV2;
3606 const now = yield* DateTime.now;
3607 const parentThreadId = ThreadId.make("thread:followup-compact-parent");
3608 const projectId = ProjectId.make("project:followup-compact");
3609 const parentRunId = RunId.make("run:followup-compact-parent");
3610 const rootNodeId = NodeId.make("node:followup-compact-parent");
3611 yield* seedParentWithTerminalTask({
3612 threadId: parentThreadId,
3613 projectId,
3614 runId: parentRunId,
3615 rootNodeId,
3616 taskId: NodeId.make("node:followup-compact-seed"),
3617 deliveryState: "acknowledged",
3618 now,
3619 });
3620 const child = yield* seedRestartCancelledChild({
3621 parentThreadId,
3622 projectId,
3623 parentRunId,
3624 rootNodeId,
3625 name: "followup-compact-child",
3626 completionWake: "always",
3627 continuationPending: false,
3628 runStatus: "completed",
3629 now,
3630 });
3631 yield* orchestrator.recoverDelegatedTasks;
3632 const tasksBefore = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents
3633 .length;
3634
3635 const send = (name: string, text: string) =>
3636 orchestrator.dispatch({
3637 type: "message.dispatch",
3638 createdBy: "user",
3639 creationSource: "web",
3640 commandId: CommandId.make(`command:followup-compact-${name}`),
3641 threadId: child.childThreadId,
3642 messageId: MessageId.make(`message:followup-compact-${name}`),
3643 text,
3644 attachments: [],
3645 dispatchMode: { type: "queue_after_active" },
3646 });
3647 yield* send("compact", "/compact");
3648 yield* send("ask", "Write the next packet.");
3649
3650 const runs = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs;
3651 const runOf = (name: string) =>
3652 runs.find((run) => run.userMessageId === `message:followup-compact-${name}`);
3653 assert.isDefined(runOf("compact"));
3654 assert.isUndefined(runOf("compact")?.delegatedTaskId);
3655 assert.isDefined(runOf("ask")?.delegatedTaskId);
3656 assert.equal(
3657 (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.length,
3658 tasksBefore + 1,
3659 );
3660 }),
3661 );
3662
3663 it.effect("settles a follow-up when its own work ends while another's still runs", () =>
3664 Effect.gen(function* () {
3665 const orchestrator = yield* Orchestrator.OrchestratorV2;
3666 const eventSink = yield* EventSink.EventSinkV2;
3667 const now = yield* DateTime.now;
3668 const parentThreadId = ThreadId.make("thread:followup-own-work-parent");
3669 const projectId = ProjectId.make("project:followup-own-work");
3670 const parentRunId = RunId.make("run:followup-own-work-parent");
3671 const rootNodeId = NodeId.make("node:followup-own-work-parent");
3672 yield* seedParentWithTerminalTask({
3673 threadId: parentThreadId,
3674 projectId,
3675 runId: parentRunId,
3676 rootNodeId,
3677 taskId: NodeId.make("node:followup-own-work-seed"),
3678 deliveryState: "acknowledged",
3679 now,
3680 });
3681 const child = yield* seedRestartCancelledChild({
3682 parentThreadId,
3683 projectId,
3684 parentRunId,
3685 rootNodeId,
3686 name: "followup-own-work-child",
3687 completionWake: "always",
3688 continuationPending: false,
3689 runStatus: "completed",
3690 now,
3691 });
3692 yield* orchestrator.recoverDelegatedTasks;
3693 const parent = yield* orchestrator.getThreadProjection(parentThreadId);
3694 const childProviderThread = {
3695 ...parent.providerThreads[0]!,
3696 id: ProviderThreadId.make("provider-thread:followup-own-work-child"),
3697 appThreadId: child.childThreadId,
3698 };
3699 const awaited = (taskId: string) => ({
3700 taskId,
3701 kind: "command" as const,
3702 awaited: true as const,
3703 });
3704 const roster = (name: string, runId: RunId, tasks: ReadonlyArray<string>) => ({
3705 id: EventId.make(`event:followup-own-work-roster-${name}`),
3706 type: "provider-thread.updated" as const,
3707 threadId: child.childThreadId,
3708 runId,
3709 driver,
3710 providerInstanceId: modelSelection.instanceId,
3711 occurredAt: now,
3712 payload: { ...childProviderThread, pendingBackgroundTasks: tasks.map(awaited) },
3713 });
3714 const settledTask = (taskId: NodeId, afterSequence: number) =>
3715 eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
3716 Stream.filter(
3717 (stored) =>
3718 stored.event.type === "subagent.updated" &&
3719 stored.event.payload.id === taskId &&
3720 stored.event.payload.status === "completed",
3721 ),
3722 Stream.take(1),
3723 Stream.runDrain,
3724 );
3725 const send = (name: string) =>
3726 Effect.gen(function* () {
3727 const messageId = MessageId.make(`message:followup-own-work-send-${name}`);
3728 yield* orchestrator.dispatch({
3729 type: "message.dispatch",
3730 createdBy: "agent",
3731 creationSource: "mcp",
3732 commandId: CommandId.make(`command:followup-own-work-send-${name}`),
3733 threadId: child.childThreadId,
3734 senderThreadId: parentThreadId,
3735 messageId,
3736 text: `Start build ${name} in the background and wait on it.`,
3737 attachments: [],
3738 dispatchMode: { type: "queue_after_active" },
3739 });
3740 return (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
3741 (run) => run.userMessageId === messageId,
3742 )!;
3743 });
3744
3745 // Follow-up A starts awaited build a, answers and ends its turn.
3746 const runA = yield* send("a");
3747 yield* eventSink.write({
3748 commandId: reconcileCommandId("followup-own-work-a-ended"),
3749 events: [
3750 roster("a", runA.id, ["a"]),
3751 {
3752 id: EventId.make("event:followup-own-work-a-answer"),
3753 type: "message.updated",
3754 threadId: child.childThreadId,
3755 runId: runA.id,
3756 occurredAt: now,
3757 payload: {
3758 id: MessageId.make("message:followup-own-work-a-answer"),
3759 threadId: child.childThreadId,
3760 runId: runA.id,
3761 nodeId: null,
3762 role: "assistant",
3763 text: "A-DONE",
3764 attachments: [],
3765 streaming: false,
3766 createdBy: "agent",
3767 creationSource: "server",
3768 createdAt: now,
3769 updatedAt: now,
3770 },
3771 },
3772 {
3773 id: EventId.make("event:followup-own-work-a-completed"),
3774 type: "run.updated",
3775 threadId: child.childThreadId,
3776 runId: runA.id,
3777 occurredAt: now,
3778 payload: { ...runA, status: "completed", startedAt: now, completedAt: now },
3779 },
3780 ],
3781 });
3782 yield* orchestrator.recoverDelegatedTasks;
3783
3784 // Follow-up B starts awaited build b and is still running.
3785 const runB = yield* send("b");
3786 yield* eventSink.write({
3787 commandId: reconcileCommandId("followup-own-work-b-started"),
3788 events: [
3789 {
3790 id: EventId.make("event:followup-own-work-b-running"),
3791 type: "run.updated",
3792 threadId: child.childThreadId,
3793 runId: runB.id,
3794 occurredAt: now,
3795 payload: { ...runB, status: "running", startedAt: now },
3796 },
3797 roster("ab", runB.id, ["a", "b"]),
3798 ],
3799 });
3800 // Run updates react in order, so the barrier settling means B's start was handled too.
3801 const barrier = yield* seedRestartCancelledChild({
3802 parentThreadId,
3803 projectId,
3804 parentRunId,
3805 rootNodeId,
3806 name: "followup-own-work-barrier",
3807 completionWake: "always",
3808 continuationPending: false,
3809 runStatus: "completed",
3810 now,
3811 });
3812 const beforeBarrier = yield* eventSink.latestSequence();
3813 yield* eventSink.write({
3814 events: [
3815 {
3816 ...runEvent({
3817 threadId: barrier.childThreadId,
3818 runId: barrier.childRunId,
3819 ordinal: 1,
3820 status: "completed",
3821 now,
3822 }),
3823 id: EventId.make("event:followup-own-work-barrier-ended"),
3824 },
3825 ],
3826 });
3827 yield* settledTask(barrier.taskId, beforeBarrier);
3828 assert.equal(
3829 (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3830 (task) => task.id === runA.delegatedTaskId,
3831 )?.status,
3832 "running",
3833 );
3834
3835 // Build a ends; the roster still holds b, and no run update follows.
3836 const beforeSettle = yield* eventSink.latestSequence();
3837 yield* eventSink.write({
3838 commandId: reconcileCommandId("followup-own-work-a-settled"),
3839 events: [roster("b", runB.id, ["b"])],
3840 });
3841 yield* settledTask(runA.delegatedTaskId!, beforeSettle);
3842 const tasks = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents;
3843 assert.equal(tasks.find((task) => task.id === runA.delegatedTaskId)?.result, "A-DONE");
3844 assert.equal(tasks.find((task) => task.id === runB.delegatedTaskId)?.status, "running");
3845 }),
3846 );
3847
3848 it.effect("waits for a run the follow-up's child queued on its own thread", () =>
3849 Effect.gen(function* () {
3850 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
3851 const followup = yield* startSelfQueuedFollowup("self-queued-followup");
3852 yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
3853 const waiting = yield* followup.readFollowup;
3854 assert.isNull(
3855 waiting.result,
3856 "the follow-up task settled on its handoff turn while the run its child queued was pending",
3857 );
3858 assert.notEqual(waiting.status, "completed");
3859
3860 yield* followup.finishRun("final", followup.selfQueuedRun, "FINAL-D");
3861 const finished = yield* followup.readFollowup;
3862 assert.equal(finished.status, "completed");
3863 assert.equal(finished.result, "FINAL-D");
3864 assert.equal(finished.completionDelivery?.state, "claimed");
3865 let wake = yield* requests.take;
3866 while (
3867 wake.threadId !== followup.parentThreadId ||
3868 wake.delegatedCompletion?.messageId === followup.firstWake.delegatedCompletion?.messageId
3869 ) {
3870 wake = yield* requests.take;
3871 }
3872 assert.equal(wake.delegatedCompletion?.parentRunId, followup.parentRunId);
3873 }),
3874 );
3875
3876 it.effect("keeps a follow-up running when the run its child queued is removed early", () =>
3877 Effect.gen(function* () {
3878 const followup = yield* startSelfQueuedFollowup("self-queued-removed-early-followup");
3879 yield* followup.removeSelfQueuedRun;
3880 const running = yield* followup.readFollowup;
3881 assert.isNull(running.result);
3882 assert.equal(running.status, "running");
3883
3884 yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
3885 const finished = yield* followup.readFollowup;
3886 assert.equal(finished.status, "completed");
3887 assert.equal(finished.result, "HANDOFF-C");
3888 }),
3889 );
3890
3891 it.effect.each(["removed", "held"] as const)(
3892 "settles a follow-up on its last turn when the run it queued is %s",
3893 (ending) =>
3894 Effect.gen(function* () {
3895 const eventSink = yield* EventSink.EventSinkV2;
3896 const now = yield* DateTime.now;
3897 const followup = yield* startSelfQueuedFollowup(`self-queued-${ending}-followup`);
3898 if (ending === "held") {
3899 yield* eventSink.write({
3900 commandId: reconcileCommandId("self-queued-held-followup-hold"),
3901 events: [
3902 {
3903 id: EventId.make("event:self-queued-held-followup-hold"),
3904 type: "run.updated",
3905 threadId: followup.childThreadId,
3906 runId: followup.selfQueuedRun.id,
3907 occurredAt: now,
3908 payload: { ...followup.selfQueuedRun, queueHeld: true },
3909 },
3910 ],
3911 });
3912 }
3913 yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
3914 if (ending === "removed") yield* followup.removeSelfQueuedRun;
3915 const finished = yield* followup.readFollowup;
3916 assert.equal(finished.status, "completed");
3917 assert.equal(finished.result, "HANDOFF-C");
3918 }),
3919 );
3920
3921 it.effect("waits for every run its child queued when a later one is moved ahead", () =>
3922 Effect.gen(function* () {
3923 const orchestrator = yield* Orchestrator.OrchestratorV2;
3924 const followup = yield* startSelfQueuedFollowup("self-queued-reordered-followup");
3925 const laterMessageId = MessageId.make("message:self-queued-reordered-followup-later");
3926 yield* orchestrator.dispatch({
3927 type: "message.dispatch",
3928 createdBy: "agent",
3929 creationSource: "mcp",
3930 commandId: CommandId.make("command:self-queued-reordered-followup-later"),
3931 threadId: followup.childThreadId,
3932 senderThreadId: followup.childThreadId,
3933 messageId: laterMessageId,
3934 text: "Do this part first.",
3935 attachments: [],
3936 dispatchMode: { type: "queue_after_active" },
3937 });
3938 const later = (yield* orchestrator.getThreadProjection(followup.childThreadId)).runs.find(
3939 (run) => run.userMessageId === laterMessageId,
3940 )!;
3941 yield* orchestrator.dispatch({
3942 type: "queued-run.reorder",
3943 commandId: CommandId.make("command:self-queued-reordered-followup-reorder"),
3944 threadId: followup.childThreadId,
3945 runId: later.id,
3946 beforeRunId: followup.selfQueuedRun.id,
3947 });
3948 assert.equal((yield* followup.readFollowup).status, "running");
3949 yield* followup.finishRun("handoff", followup.followupRun, "HANDOFF-C");
3950 yield* followup.finishRun("later", later, "PART-ONE");
3951 const waiting = yield* followup.readFollowup;
3952 assert.isNull(waiting.result, "the follow-up settled while an earlier queued run waited");
3953
3954 // Promotion moves a queued run after the runs that started ahead of it.
3955 const promoted = { ...followup.selfQueuedRun, ordinal: later.ordinal + 1 };
3956 yield* followup.finishRun("final", promoted, "FINAL-D");
3957 const finished = yield* followup.readFollowup;
3958 assert.equal(finished.status, "completed");
3959 assert.equal(finished.result, "FINAL-D");
3960 }),
3961 );
3962
3963 const outcomes = [
3964 "completed",
3965 "failed",
3966 "interrupted",
3967 "restarted",
3968 "stopped",
3969 "overlapping",
3970 "after-cancel",
3971 "cancelled",
3972 ] as const;
3973 it.effect.each(outcomes)(
3974 "delivers a %s follow-up independently of the original task",
3975 (outcome) =>
3976 Effect.gen(function* () {
3977 const orchestrator = yield* Orchestrator.OrchestratorV2;
3978 const eventSink = yield* EventSink.EventSinkV2;
3979 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
3980 const now = yield* DateTime.now;
3981 const parentThreadId = ThreadId.make(`thread:followup-${outcome}`);
3982 const projectId = ProjectId.make(`project:followup-${outcome}`);
3983 const parentRunId = RunId.make(`run:followup-${outcome}`);
3984 const rootNodeId = NodeId.make(`node:followup-${outcome}`);
3985 yield* seedParentWithTerminalTask({
3986 threadId: parentThreadId,
3987 projectId,
3988 runId: parentRunId,
3989 rootNodeId,
3990 taskId: NodeId.make(`node:seed-${outcome}`),
3991 deliveryState: "acknowledged",
3992 now,
3993 });
3994 const child = yield* seedRestartCancelledChild({
3995 parentThreadId,
3996 projectId,
3997 parentRunId,
3998 rootNodeId,
3999 name: `followup-child-${outcome}`,
4000 completionWake: "always",
4001 continuationPending: false,
4002 runStatus: "completed",
4003 now,
4004 });
4005 yield* orchestrator.recoverDelegatedTasks;
4006 let firstWake = yield* requests.take;
4007 while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
4008 assert.equal(firstWake.delegatedCompletion?.parentRunId, parentRunId);
4009 yield* orchestrator.dispatch({
4010 type: "delegated_task.completion-delivery.acknowledge",
4011 commandId: CommandId.make(`command:followup-ack-${outcome}`),
4012 parentThreadId,
4013 taskId: child.taskId,
4014 observedByRunId: parentRunId,
4015 });
4016 if (outcome === "after-cancel") {
4017 // A follow-up is its own task, even after task_cancel reached the original.
4018 const settled = (yield* orchestrator.getThreadProjection(
4019 parentThreadId,
4020 )).subagents.find((task) => task.id === child.taskId)!;
4021 // The follow-up starts in the same millisecond the cancel was recorded.
4022 yield* eventSink.write({
4023 commandId: reconcileCommandId("followup-original-cancelled"),
4024 events: [
4025 {
4026 id: EventId.make("event:followup-original-cancelled"),
4027 type: "subagent.updated",
4028 threadId: parentThreadId,
4029 nodeId: settled.id,
4030 occurredAt: now,
4031 payload: { ...settled, cancelledRunId: child.childRunId },
4032 },
4033 ],
4034 });
4035 }
4036 const original = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
4037 (task) => task.id === child.taskId,
4038 )!;
4039 const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
4040 (run) => run.id === parentRunId,
4041 )!;
4042 yield* eventSink.write({
4043 commandId: reconcileCommandId(`followup-parent-finished-${outcome}`),
4044 events: [
4045 {
4046 id: EventId.make(`event:followup-parent-finished-${outcome}`),
4047 type: "run.updated",
4048 threadId: parentThreadId,
4049 runId: parentRunId,
4050 occurredAt: now,
4051 payload: { ...parentRun, status: "completed", completedAt: now },
4052 },
4053 ],
4054 });
4055 const messageId = MessageId.make(`message:followup-${outcome}`);
4056 yield* orchestrator.dispatch({
4057 type: "message.dispatch",
4058 createdBy: outcome === "failed" ? "user" : "agent",
4059 creationSource: outcome === "failed" ? "web" : "mcp",
4060 commandId: CommandId.make(`command:followup-send-${outcome}`),
4061 threadId: child.childThreadId,
4062 // `t3 thread send` and unscoped MCP sessions carry no sender thread.
4063 ...(outcome === "completed" ? {} : { senderThreadId: parentThreadId }),
4064 messageId,
4065 text: "Check the same work again.",
4066 attachments: [],
4067 dispatchMode: { type: "start_immediately" },
4068 });
4069 const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId);
4070 let run = childProjection.runs.find((row) => row.userMessageId === messageId)!;
4071 assert.isDefined(run.delegatedTaskId);
4072 const workingParent = yield* orchestrator.getShellSnapshot();
4073 const backgroundTask = workingParent.threads
4074 .find((thread) => thread.id === parentThreadId)
4075 ?.pendingBackgroundTasks?.find((task) => task.kind === "subagent");
4076 assert.equal(backgroundTask?.childThreadId, child.childThreadId);
4077 if (outcome === "overlapping") {
4078 yield* orchestrator.dispatch({
4079 type: "message.dispatch",
4080 createdBy: "agent",
4081 creationSource: "mcp",
4082 commandId: CommandId.make("command:overlapping-followup"),
4083 threadId: child.childThreadId,
4084 senderThreadId: parentThreadId,
4085 messageId: MessageId.make("message:overlapping-followup"),
4086 text: "Keep working on another follow-up.",
4087 attachments: [],
4088 dispatchMode: { type: "queue_after_active" },
4089 });
4090 }
4091 if (outcome === "restarted") {
4092 const cut = { ...run, status: "cancelled" as const, startedAt: now, completedAt: now };
4093 yield* eventSink.writeWithEffects({
4094 commandId: reconcileCommandId("followup-restart-cut"),
4095 events: [
4096 {
4097 id: EventId.make("event:followup-restart-cut"),
4098 type: "run.updated",
4099 threadId: child.childThreadId,
4100 runId: run.id,
4101 occurredAt: now,
4102 payload: cut,
4103 },
4104 ],
4105 effects: [
4106 {
4107 id: `effect:restart-continuation:${run.id}`,
4108 commandId: reconcileCommandId("followup-restart-cut"),
4109 threadId: child.childThreadId,
4110 request: { type: "provider-runtime.continue", sourceRunId: run.id },
4111 },
4112 ],
4113 });
4114 yield* orchestrator.recoverDelegatedTasks;
4115 assert.isNull(
4116 (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
4117 (task) => task.id === run.delegatedTaskId,
4118 )!.result,
4119 );
4120 const continuationMessageId = MessageId.make("message:followup-restart-continuation");
4121 yield* orchestrator.dispatch({
4122 type: "message.dispatch",
4123 createdBy: "agent",
4124 creationSource: "server",
4125 commandId: CommandId.make("command:followup-restart-continuation"),
4126 threadId: child.childThreadId,
4127 restartContinuationOfRunId: run.id,
4128 messageId: continuationMessageId,
4129 text: "Continue where you left off.",
4130 attachments: [],
4131 dispatchMode: { type: "start_immediately" },
4132 });
4133 const continuation = (yield* orchestrator.getThreadProjection(
4134 child.childThreadId,
4135 )).runs.find((row) => row.userMessageId === continuationMessageId)!;
4136 assert.equal(continuation.delegatedTaskId, run.delegatedTaskId);
4137 run = continuation;
4138 }
4139 if (outcome === "stopped") {
4140 yield* orchestrator.dispatch({
4141 type: "thread.stop",
4142 threadId: parentThreadId,
4143 commandId: CommandId.make(`command:followup-stop-${outcome}`),
4144 });
4145 }
4146 if (outcome === "cancelled") {
4147 // task_cancel reaches the follow-up after its turn ended, while background work runs.
4148 yield* eventSink.write({
4149 commandId: reconcileCommandId("followup-waiting"),
4150 events: [
4151 {
4152 id: EventId.make("event:followup-waiting"),
4153 type: "run.updated",
4154 threadId: child.childThreadId,
4155 runId: run.id,
4156 occurredAt: now,
4157 payload: { ...run, status: "waiting", startedAt: now },
4158 },
4159 ],
4160 });
4161 yield* orchestrator.cancelDelegatedTask({
4162 parentThreadId,
4163 taskId: run.delegatedTaskId!,
4164 command: {
4165 type: "thread.stop",
4166 commandId: CommandId.make(`cancel-followup-${outcome}`),
4167 threadId: child.childThreadId,
4168 stopDelegatedTasks: false,
4169 },
4170 stopDescendants: () => Effect.void,
4171 });
4172 }
4173 yield* eventSink.write({
4174 commandId: reconcileCommandId(`followup-finish-${outcome}`),
4175 events: [
4176 {
4177 id: EventId.make(`event:followup-question-${outcome}`),
4178 type: "runtime-request.updated",
4179 threadId: child.childThreadId,
4180 runId: run.id,
4181 occurredAt: now,
4182 payload: {
4183 id: RuntimeRequestId.make(`request:followup-${outcome}`),
4184 nodeId: run.rootNodeId!,
4185 providerTurnId: null,
4186 nativeRequestRef: null,
4187 kind: "user_input",
4188 status: "pending",
4189 responseCapability: { type: "message" },
4190 createdAt: now,
4191 resolvedAt: null,
4192 },
4193 },
4194 {
4195 id: EventId.make(`event:followup-finish-${outcome}`),
4196 type: "run.updated",
4197 threadId: child.childThreadId,
4198 runId: run.id,
4199 occurredAt: now,
4200 payload: {
4201 ...run,
4202 status:
4203 outcome === "stopped"
4204 ? "interrupted"
4205 : outcome === "restarted" ||
4206 outcome === "overlapping" ||
4207 outcome === "after-cancel" ||
4208 outcome === "cancelled"
4209 ? "completed"
4210 : outcome,
4211 startedAt: now,
4212 completedAt: now,
4213 },
4214 },
4215 ],
4216 });
4217 yield* orchestrator.recoverDelegatedTasks;
4218 // Only an interrupted or cancelled follow-up ends its question, even after the original
4219 // was cancelled.
4220 const ended =
4221 outcome === "stopped" || outcome === "interrupted" || outcome === "cancelled";
4222 const { runtimeRequests } = yield* orchestrator.getThreadProjection(child.childThreadId);
4223 assert.equal(runtimeRequests[0]?.status, ended ? "resolved" : "pending");
4224 const finished = yield* orchestrator.getThreadProjection(parentThreadId);
4225 assert.deepEqual(
4226 finished.subagents.find((task) => task.id === child.taskId),
4227 original,
4228 );
4229 const followup = finished.subagents.find((task) => task.id === run.delegatedTaskId)!;
4230 if (outcome === "overlapping") {
4231 const later = finished.subagents.find(
4232 (task) =>
4233 task.childThreadId === child.childThreadId &&
4234 task.id !== child.taskId &&
4235 task.id !== followup.id,
4236 )!;
4237 assert.equal(later.status, "pending");
4238 assert.isNull(later.result);
4239 }
4240 assert.equal(
4241 followup.status,
4242 outcome === "stopped" || outcome === "cancelled"
4243 ? "interrupted"
4244 : outcome === "restarted" || outcome === "overlapping" || outcome === "after-cancel"
4245 ? "completed"
4246 : outcome,
4247 );
4248 assert.equal(
4249 followup.completionDelivery?.state,
4250 outcome === "stopped" || outcome === "cancelled" ? "disposed" : "claimed",
4251 );
4252 assert.equal(followup.cancelledRunId, outcome === "cancelled" ? run.id : undefined);
4253 if (outcome !== "stopped" && outcome !== "cancelled") {
4254 let wake = yield* requests.take;
4255 while (
4256 wake.threadId !== parentThreadId ||
4257 wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId
4258 ) {
4259 wake = yield* requests.take;
4260 }
4261 assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId);
4262 assert.notEqual(
4263 wake.delegatedCompletion?.messageId,
4264 firstWake.delegatedCompletion?.messageId,
4265 );
4266 }
4267 assert.equal(
4268 finished.contextTransfers.filter(
4269 (transfer) =>
4270 transfer.type === "subagent_result" &&
4271 transfer.sourceThreadId === child.childThreadId,
4272 ).length,
4273 2,
4274 );
4275 yield* orchestrator.recoverDelegatedTasks;
4276 const recovered = yield* orchestrator.getThreadProjection(parentThreadId);
4277 assert.equal(recovered.contextTransfers.length, finished.contextTransfers.length);
4278 if (outcome === "completed") {
4279 const automaticMessageId = MessageId.make("message:followup-automatic-wake");
4280 yield* orchestrator.dispatch({
4281 type: "message.dispatch",
4282 createdBy: "agent",
4283 creationSource: "server",
4284 commandId: CommandId.make("command:followup-automatic-wake"),
4285 threadId: child.childThreadId,
4286 messageId: automaticMessageId,
4287 text: "A monitor reported new output.",
4288 attachments: [],
4289 notification: {
4290 source: { kind: "monitor" },
4291 outcome: "updated",
4292 summary: "Monitor updated",
4293 },
4294 dispatchMode: { type: "queue_after_active" },
4295 });
4296 const automatic = (yield* orchestrator.getThreadProjection(
4297 child.childThreadId,
4298 )).runs.find((run) => run.userMessageId === automaticMessageId)!;
4299 assert.isUndefined(automatic.delegatedTaskId);
4300 yield* eventSink.write({
4301 commandId: reconcileCommandId("followup-automatic-finished"),
4302 events: [
4303 {
4304 id: EventId.make("event:followup-automatic-finished"),
4305 type: "run.updated",
4306 threadId: child.childThreadId,
4307 runId: automatic.id,
4308 occurredAt: now,
4309 payload: { ...automatic, status: "completed", startedAt: now, completedAt: now },
4310 },
4311 ],
4312 });
4313 yield* orchestrator.recoverDelegatedTasks;
4314 const afterAutomatic = yield* orchestrator.getThreadProjection(parentThreadId);
4315 assert.deepEqual(afterAutomatic.subagents, recovered.subagents);
4316 assert.equal(afterAutomatic.contextTransfers.length, recovered.contextTransfers.length);
4317 const control = yield* seedRestartCancelledChild({
4318 parentThreadId,
4319 projectId,
4320 parentRunId,
4321 rootNodeId,
4322 name: "followup-control-child",
4323 completionWake: "always",
4324 continuationPending: false,
4325 runStatus: "completed",
4326 now,
4327 });
4328 yield* orchestrator.recoverDelegatedTasks;
4329 const afterControl = yield* orchestrator.getThreadProjection(parentThreadId);
4330 assert.equal(
4331 afterControl.subagents.find((task) => task.id === control.taskId)?.completionDelivery
4332 ?.state,
4333 "claimed",
4334 );
4335 assert.equal(
4336 afterControl.contextTransfers.filter(
4337 (transfer) =>
4338 transfer.type === "subagent_result" &&
4339 transfer.sourceThreadId === control.childThreadId,
4340 ).length,
4341 1,
4342 );
4343 }
4344 }),
4345 );
4346
4347 it.effect("tracks a child's usage-limit auto-resume as a follow-up", () =>
4348 Effect.gen(function* () {
4349 const orchestrator = yield* Orchestrator.OrchestratorV2;
4350 const eventSink = yield* EventSink.EventSinkV2;
4351 const now = yield* DateTime.now;
4352 const parentThreadId = ThreadId.make("thread:limit-resume-parent");
4353 const projectId = ProjectId.make("project:limit-resume");
4354 const parentRunId = RunId.make("run:limit-resume-parent");
4355 const rootNodeId = NodeId.make("node:limit-resume-parent");
4356 yield* seedParentWithTerminalTask({
4357 threadId: parentThreadId,
4358 projectId,
4359 runId: parentRunId,
4360 rootNodeId,
4361 taskId: NodeId.make("node:limit-resume-seed"),
4362 deliveryState: "acknowledged",
4363 now,
4364 });
4365 const child = yield* seedRestartCancelledChild({
4366 parentThreadId,
4367 projectId,
4368 parentRunId,
4369 rootNodeId,
4370 name: "limit-resume-child",
4371 completionWake: "always",
4372 continuationPending: false,
4373 runStatus: "completed",
4374 now,
4375 });
4376 const childRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs[0]!;
4377 yield* eventSink.write({
4378 commandId: reconcileCommandId("limit-resume-child-limited"),
4379 events: [
4380 {
4381 id: EventId.make("event:limit-resume-child-failed"),
4382 type: "run.updated",
4383 threadId: child.childThreadId,
4384 runId: childRun.id,
4385 occurredAt: now,
4386 payload: { ...childRun, status: "failed", completedAt: now },
4387 },
4388 {
4389 id: EventId.make("event:limit-resume-child-error"),
4390 type: "turn-item.updated",
4391 threadId: child.childThreadId,
4392 occurredAt: now,
4393 payload: {
4394 id: TurnItemId.make("turn-item:limit-resume-child-error"),
4395 type: "error",
4396 threadId: child.childThreadId,
4397 runId: childRun.id,
4398 nodeId: childRun.rootNodeId,
4399 providerThreadId: null,
4400 providerTurnId: null,
4401 nativeItemRef: null,
4402 parentItemId: null,
4403 ordinal: 1,
4404 status: "failed",
4405 title: "Usage limit reached",
4406 startedAt: now,
4407 completedAt: now,
4408 updatedAt: now,
4409 failure: {
4410 class: "usage_limit",
4411 message: "Plan limit reached.",
4412 code: "usageLimitExceeded",
4413 retryable: null,
4414 resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 1 })),
4415 },
4416 },
4417 },
4418 ],
4419 });
4420 yield* orchestrator.recoverDelegatedTasks;
4421 const childShell = Effect.map(orchestrator.getShellSnapshot(), (snapshot) =>
4422 snapshot.threads.find((thread) => thread.id === child.childThreadId),
4423 );
4424 const arm = limitRecoveryCommand((yield* childShell)!, true, DateTime.toEpochMillis(now));
4425 assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);
4426 yield* orchestrator.autoArmLimitRecovery({
4427 commandId: arm.commandId,
4428 threadId: arm.threadId,
4429 limitRecovery: arm.limitRecovery,
4430 });
4431 yield* TestClock.adjust("1 minute");
4432 const resume = limitRecoveryCommand(
4433 (yield* childShell)!,
4434 true,
4435 DateTime.toEpochMillis(yield* DateTime.now),
4436 );
4437 assert(resume?.type === "message.dispatch");
4438 yield* orchestrator.dispatch(resume);
4439
4440 const resumed = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
4441 (run) => run.userMessageId === resume.messageId,
4442 );
4443 assert.isDefined(resumed?.delegatedTaskId);
4444 const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
4445 (task) => task.id === resumed?.delegatedTaskId,
4446 );
4447 assert.equal(followup?.childThreadId, child.childThreadId);
4448 assert.isNull(followup?.result);
4449 }),
4450 );
4451
4452 it.effect("serializes a recovered child answer with its parent and tracks the follow-up", () =>
4453 Effect.gen(function* () {
4454 const orchestrator = yield* Orchestrator.OrchestratorV2;
4455 const eventSink = yield* EventSink.EventSinkV2;
4456 const now = yield* DateTime.now;
4457 const parentThreadId = ThreadId.make("thread:question-answer-parent");
4458 const projectId = ProjectId.make("project:question-answer");
4459 const parentRunId = RunId.make("run:question-answer-parent");
4460 const rootNodeId = NodeId.make("node:question-answer-parent");
4461 yield* seedParentWithTerminalTask({
4462 threadId: parentThreadId,
4463 projectId,
4464 runId: parentRunId,
4465 rootNodeId,
4466 taskId: NodeId.make("node:question-answer-seed"),
4467 deliveryState: "acknowledged",
4468 now,
4469 });
4470 const child = yield* seedRestartCancelledChild({
4471 parentThreadId,
4472 projectId,
4473 parentRunId,
4474 rootNodeId,
4475 name: "question-answer-child",
4476 completionWake: "always",
4477 continuationPending: false,
4478 now,
4479 });
4480 // The cancelled result is published before anyone answers the question.
4481 yield* orchestrator.recoverDelegatedTasks;
4482 assert.isNotNull(
4483 (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
4484 (task) => task.id === child.taskId,
4485 )?.result,
4486 );
4487 const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
4488 (run) => run.id === parentRunId,
4489 )!;
4490 yield* eventSink.write({
4491 events: [
4492 {
4493 id: EventId.make("event:question-answer-parent-completed"),
4494 type: "run.updated",
4495 threadId: parentThreadId,
4496 runId: parentRunId,
4497 occurredAt: now,
4498 payload: { ...parentRun, status: "completed", completedAt: now },
4499 },
4500 ],
4501 });
4502 const requestId = RuntimeRequestId.make("request:question-answer");
4503 const nodeId = NodeId.make("node:question-answer-request");
4504 yield* eventSink.write({
4505 commandId: CommandId.make("command:seed-question-answer"),
4506 events: [
4507 {
4508 id: EventId.make("event:question-answer-request"),
4509 type: "runtime-request.updated",
4510 threadId: child.childThreadId,
4511 nodeId,
4512 occurredAt: now,
4513 payload: {
4514 id: requestId,
4515 nodeId,
4516 providerTurnId: null,
4517 nativeRequestRef: null,
4518 kind: "user_input",
4519 status: "pending",
4520 responseCapability: { type: "message" },
4521 createdAt: now,
4522 resolvedAt: null,
4523 },
4524 },
4525 {
4526 id: EventId.make("event:question-answer-item"),
4527 type: "turn-item.updated",
4528 threadId: child.childThreadId,
4529 nodeId,
4530 occurredAt: now,
4531 payload: {
4532 id: TurnItemId.make("turn-item:question-answer"),
4533 type: "user_input_request",
4534 threadId: child.childThreadId,
4535 runId: child.childRunId,
4536 nodeId,
4537 providerThreadId: null,
4538 providerTurnId: null,
4539 nativeItemRef: null,
4540 parentItemId: null,
4541 ordinal: 0,
4542 status: "waiting",
4543 title: null,
4544 startedAt: now,
4545 completedAt: null,
4546 updatedAt: now,
4547 requestId,
4548 questions: [
4549 { id: "color", header: "Color", question: "Which color?", options: [] },
4550 ],
4551 },
4552 },
4553 ],
4554 });
4555
4556 const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
4557 const parentHeld = yield* Deferred.make<void>();
4558 const releaseParent = yield* Deferred.make<void>();
4559 const answerWaiting = yield* Deferred.make<void>();
4560 const withLock = executor.withLock;
4561 const parentMutation = yield* withLock(
4562 parentThreadId,
4563 Deferred.succeed(parentHeld, undefined).pipe(
4564 Effect.andThen(Deferred.await(releaseParent)),
4565 ),
4566 ).pipe(Effect.forkScoped({ startImmediately: true }));
4567 yield* Deferred.await(parentHeld);
4568 const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = (
4569 key,
4570 effect,
4571 ) =>
4572 key === parentThreadId
4573 ? Deferred.succeed(answerWaiting, undefined).pipe(Effect.andThen(withLock(key, effect)))
4574 : withLock(key, effect);
4575 const spy = vi.spyOn(executor, "withLock").mockImplementation(observeLock);
4576 yield* Effect.gen(function* () {
4577 const answer = yield* orchestrator
4578 .dispatch({
4579 type: "runtime-request.respond",
4580 commandId: CommandId.make("command:question-answer"),
4581 threadId: child.childThreadId,
4582 requestId,
4583 answers: { color: "Blue" },
4584 })
4585 .pipe(Effect.forkScoped({ startImmediately: true }));
4586 yield* Effect.raceFirst(Deferred.await(answerWaiting), Fiber.join(answer));
4587 const waiting = yield* orchestrator.getThreadProjection(child.childThreadId);
4588 assert.equal(
4589 waiting.runtimeRequests.find((row) => row.id === requestId)?.status,
4590 "pending",
4591 );
4592 assert.isFalse((yield* executor.activeKeys).includes(child.childThreadId));
4593 yield* Deferred.succeed(releaseParent, undefined);
4594 yield* Fiber.join(parentMutation);
4595 yield* Fiber.join(answer);
4596 }).pipe(Effect.ensuring(Effect.sync(() => spy.mockRestore())));
4597
4598 const answerRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
4599 (run) => run.userMessageId === MessageId.make(`async-answer:${requestId}`),
4600 );
4601 assert.isDefined(answerRun?.delegatedTaskId);
4602 const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
4603 (task) => task.id === answerRun?.delegatedTaskId,
4604 );
4605 assert.equal(followup?.childThreadId, child.childThreadId);
4606 assert.isNull(followup?.result);
4607 assert.isTrue(
4608 (yield* orchestrator.getThreadShell(parentThreadId))?.pendingBackgroundTasks?.some(
4609 (task) => task.kind === "subagent" && task.childThreadId === child.childThreadId,
4610 ),
4611 );
4612 }),
4613 );
4614 },
4615 );
4616
4617 it.layer(layerTest)("delegated tasks across a server restart", (it) => {
4618 it.effect.each([
4619 { name: "releases", key: "", pendingBackgroundTasks: [], released: true },
4620 {
4621 name: "keeps while an unawaited background command runs",
4622 key: "-background",
4623 pendingBackgroundTasks: [{ taskId: "dev-server", kind: "command" as const }],
4624 released: false,
4625 },
4626 ])(
4627 "queues a child result without creating a provider history handoff and $name its process",
4628 ({ key, pendingBackgroundTasks, released }) =>
4629 Effect.gen(function* () {
4630 const orchestrator = yield* Orchestrator.OrchestratorV2;
4631 const eventSink = yield* EventSink.EventSinkV2;
4632 const now = yield* DateTime.now;
4633 const threadId = ThreadId.make(`thread:result-message${key}-parent`);
4634 const projectId = ProjectId.make(`project:result-message${key}`);
4635 const runId = RunId.make(`run:result-message${key}-parent`);
4636 const rootNodeId = NodeId.make(`node:result-message${key}-parent`);
4637 yield* seedParentWithTerminalTask({
4638 threadId,
4639 projectId,
4640 runId,
4641 rootNodeId,
4642 taskId: NodeId.make(`node:result-message${key}-seed`),
4643 deliveryState: "acknowledged",
4644 now,
4645 });
4646 const child = yield* seedRestartCancelledChild({
4647 parentThreadId: threadId,
4648 projectId,
4649 parentRunId: runId,
4650 rootNodeId,
4651 name: `result-message${key}-child`,
4652 completionWake: "always",
4653 continuationPending: false,
4654 runStatus: "completed",
4655 now,
4656 });
4657 const parent = yield* orchestrator.getThreadProjection(threadId);
4658 const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId);
4659 const providerThreadId = ProviderThreadId.make(
4660 `provider-thread:result-message${key}-child`,
4661 );
4662 yield* eventSink.write({
4663 commandId: CommandId.make(`command:result-message${key}-child-provider`),
4664 events: [
4665 {
4666 id: EventId.make(`event:result-message${key}-child-session`),
4667 type: "provider-session.attached",
4668 threadId: child.childThreadId,
4669 occurredAt: now,
4670 payload: {
4671 id: ProviderSessionId.make(`provider-session:result-message${key}-child`),
4672 driver,
4673 providerInstanceId: modelSelection.instanceId,
4674 status: "ready",
4675 cwd: "/repo",
4676 model: modelSelection.model,
4677 capabilities: {
4678 ...CodexProviderCapabilitiesV2,
4679 sessions: {
4680 ...CodexProviderCapabilitiesV2.sessions,
4681 supportsMultipleProviderThreadsPerSession: false,
4682 },
4683 },
4684 createdAt: now,
4685 updatedAt: now,
4686 lastError: null,
4687 },
4688 },
4689 {
4690 id: EventId.make(`event:result-message${key}-child-provider`),
4691 type: "provider-thread.updated",
4692 threadId: child.childThreadId,
4693 occurredAt: now,
4694 payload: {
4695 ...parent.providerThreads[0]!,
4696 id: providerThreadId,
4697 appThreadId: child.childThreadId,
4698 pendingBackgroundTasks,
4699 },
4700 },
4701 {
4702 id: EventId.make(`event:result-message${key}-child-run`),
4703 type: "run.updated",
4704 threadId: child.childThreadId,
4705 runId: child.childRunId,
4706 occurredAt: now,
4707 payload: { ...childProjection.runs[0]!, providerThreadId },
4708 },
4709 ],
4710 });
4711
4712 yield* orchestrator.recoverDelegatedTasks;
4713 const recovered = yield* orchestrator.getThreadProjection(threadId);
4714 const task = recovered.subagents.find((candidate) => candidate.id === child.taskId)!;
4715 assert.equal(task.status, "completed");
4716 assert.isNotNull(task.result);
4717 assert.isEmpty(recovered.contextHandoffs);
4718 assert.equal(recovered.contextTransfers.length, 1);
4719 // A one-process-per-thread child releases its process once its result is recorded,
4720 // unless background work it did not await would die with it.
4721 assert.equal(
4722 (yield* orchestrator.getThreadProjection(child.childThreadId)).providerSessions.length,
4723 released ? 0 : 1,
4724 );
4725
4726 const delivery = recovered.runs.find((candidate) => candidate.id === runId)!
4727 .delegatedCompletion!.delivery!;
4728 const messageId = delivery.messageId;
4729 yield* orchestrator.dispatch({
4730 type: "message.dispatch",
4731 commandId: CommandId.make(`command:result-message${key}-delivery`),
4732 threadId,
4733 messageId,
4734 text: "Delegated task finished.",
4735 attachments: [],
4736 dispatchMode: { type: "queue_after_active" },
4737 createdBy: "agent",
4738 creationSource: "server",
4739 delegatedCompletion: {
4740 parentRunId: runId,
4741 generation: delivery.generation,
4742 taskIds: [child.taskId],
4743 },
4744 });
4745 const queued = yield* orchestrator.getThreadProjection(threadId);
4746 assert.include(
4747 queued.messages.find((message) => message.id === messageId)?.text ?? "",
4748 task.result!,
4749 );
4750 }),
4751 );
4752
4753 it.effect("holds a restart-cancelled child for its continuation's result", () =>
4754 Effect.gen(function* () {
4755 const orchestrator = yield* Orchestrator.OrchestratorV2;
4756 const eventSink = yield* EventSink.EventSinkV2;
4757 const now = yield* DateTime.now;
4758 const threadId = ThreadId.make("thread:restart-parent");
4759 const projectId = ProjectId.make("project:restart-parent");
4760 const runId = RunId.make("run:restart-parent");
4761 const rootNodeId = NodeId.make("node:restart-parent-root");
4762 yield* seedParentWithTerminalTask({
4763 threadId,
4764 projectId,
4765 runId,
4766 rootNodeId,
4767 taskId: NodeId.make("node:restart-parent-settled"),
4768 deliveryState: "delivered",
4769 now,
4770 });
4771 const child = (
4772 name: string,
4773 continuationPending: boolean,
4774 runStatus?: "cancelled" | "completed",
4775 ) =>
4776 seedRestartCancelledChild({
4777 parentThreadId: threadId,
4778 projectId,
4779 parentRunId: runId,
4780 rootNodeId,
4781 name,
4782 completionWake: "always",
4783 continuationPending,
4784 ...(runStatus === undefined ? {} : { runStatus }),
4785 now,
4786 });
4787 const resumed = yield* child("restart-resumed-child", true);
4788 const stopped = yield* child("restart-stopped-child", false);
4789 const backgrounded = yield* child("restart-backgrounded-child", true, "completed");
4790 const recut = yield* child("restart-recut-child", false);
4791 const recutContinuationId = RunId.make("run:restart-recut-child:2");
4792 const recutCommandId = CommandId.make("command:restart-recut-child:reconcile");
4793 const recutRun = runEvent({
4794 threadId: recut.childThreadId,
4795 runId: recutContinuationId,
4796 ordinal: 2,
4797 status: "cancelled",
4798 now,
4799 });
4800 yield* eventSink.writeWithEffects({
4801 commandId: recutCommandId,
4802 events: [
4803 {
4804 ...recutRun,
4805 payload: {
4806 ...recutRun.payload,
4807 startedAt: null,
4808 restartContinuationOfRunId: recut.childRunId,
4809 },
4810 },
4811 ],
4812 effects: [
4813 {
4814 id: `effect:restart-continuation:${recutContinuationId}`,
4815 commandId: recutCommandId,
4816 threadId: recut.childThreadId,
4817 request: { type: "provider-runtime.continue", sourceRunId: recutContinuationId },
4818 },
4819 ],
4820 });
4821
4822 yield* orchestrator.recoverDelegatedTasks;
4823
4824 const recovered = yield* orchestrator.getThreadProjection(threadId);
4825 const task = (id: NodeId) => recovered.subagents.find((row) => row.id === id);
4826 assert.equal(task(stopped.taskId)?.status, "cancelled");
4827 assert.equal(task(stopped.taskId)?.completionDelivery?.state, "claimed");
4828 assert.equal(task(resumed.taskId)?.status, "running");
4829 assert.isNull(task(resumed.taskId)?.result ?? null);
4830 assert.equal(task(backgrounded.taskId)?.status, "running");
4831 assert.isNull(task(backgrounded.taskId)?.result ?? null);
4832 assert.equal(task(recut.taskId)?.status, "running");
4833 assert.isTrue(yield* orchestrator.delegatedTaskResultPending(recut.childThreadId));
4834 assert.isTrue(yield* orchestrator.delegatedTaskResultPending(resumed.childThreadId));
4835 assert.isFalse(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
4836 yield* orchestrator.recoverDelegatedTask(recut.childThreadId, {
4837 sourceRunId: recut.childRunId,
4838 });
4839 const replayed = yield* orchestrator.getThreadProjection(threadId);
4840 assert.equal(replayed.subagents.find((row) => row.id === recut.taskId)?.status, "running");
4841 yield* eventSink.write({
4842 commandId: CommandId.make("command:restart-stopped-child:resumed"),
4843 events: [
4844 runEvent({
4845 threadId: stopped.childThreadId,
4846 runId: RunId.make("run:restart-stopped-child:2"),
4847 ordinal: 2,
4848 status: "running",
4849 now,
4850 }),
4851 ],
4852 });
4853 assert.isTrue(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
4854 assert.isFalse(
4855 recovered.contextTransfers.some(
4856 (transfer) => transfer.sourceThreadId === resumed.childThreadId,
4857 ),
4858 );
4859
4860 const afterSequence = yield* eventSink.latestSequence();
4861 const continuationRunId = RunId.make("run:restart-resumed-child:2");
4862 yield* eventSink.write({
4863 commandId: CommandId.make("command:restart-resumed-child:completed"),
4864 events: [
4865 {
4866 id: EventId.make("event:restart-resumed-child:result"),
4867 type: "message.updated",
4868 threadId: resumed.childThreadId,
4869 runId: continuationRunId,
4870 occurredAt: now,
4871 payload: {
4872 id: MessageId.make("message:restart-resumed-child:result"),
4873 threadId: resumed.childThreadId,
4874 runId: continuationRunId,
4875 nodeId: null,
4876 role: "assistant",
4877 text: "Finished after the restart.",
4878 attachments: [],
4879 streaming: false,
4880 createdBy: "agent",
4881 creationSource: "server",
4882 createdAt: now,
4883 updatedAt: now,
4884 },
4885 },
4886 runEvent({
4887 threadId: resumed.childThreadId,
4888 runId: continuationRunId,
4889 ordinal: 2,
4890 status: "completed",
4891 now,
4892 }),
4893 ],
4894 });
4895 const settled = yield* eventSink
4896 .stream({ afterSequence, eventType: "subagent.updated" })
4897 .pipe(
4898 Stream.filter(
4899 (stored) =>
4900 stored.event.type === "subagent.updated" &&
4901 stored.event.payload.id === resumed.taskId,
4902 ),
4903 Stream.take(1),
4904 Stream.runHead,
4905 );
4906 assert.isTrue(settled._tag === "Some");
4907 const finished = yield* orchestrator.getThreadProjection(threadId);
4908 const finishedTask = finished.subagents.find((row) => row.id === resumed.taskId);
4909 assert.equal(finishedTask?.status, "completed");
4910 assert.equal(finishedTask?.result, "Finished after the restart.");
4911 }),
4912 );
4913
4914 it.effect("settles a restart-cancelled child whose continuation declines to start", () =>
4915 Effect.gen(function* () {
4916 const orchestrator = yield* Orchestrator.OrchestratorV2;
4917 const now = yield* DateTime.now;
4918 const threadId = ThreadId.make("thread:restart-declined-parent");
4919 const projectId = ProjectId.make("project:restart-declined-parent");
4920 const runId = RunId.make("run:restart-declined-parent");
4921 const rootNodeId = NodeId.make("node:restart-declined-parent-root");
4922 yield* seedParentWithTerminalTask({
4923 threadId,
4924 projectId,
4925 runId,
4926 rootNodeId,
4927 taskId: NodeId.make("node:restart-declined-parent-settled"),
4928 deliveryState: "delivered",
4929 now,
4930 });
4931 const child = yield* seedRestartCancelledChild({
4932 parentThreadId: threadId,
4933 projectId,
4934 parentRunId: runId,
4935 rootNodeId,
4936 name: "restart-declined-child",
4937 completionWake: "always",
4938 continuationPending: true,
4939 now,
4940 });
4941 yield* orchestrator.recoverDelegatedTasks;
4942 const held = yield* orchestrator.getThreadProjection(threadId);
4943 assert.equal(held.subagents.find((row) => row.id === child.taskId)?.status, "running");
4944
4945 yield* continueRestartedRun({
4946 threadId: child.childThreadId,
4947 sourceRunId: child.childRunId,
4948 }).pipe(
4949 Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false })),
4950 );
4951
4952 const settled = yield* orchestrator.getThreadProjection(threadId);
4953 const task = settled.subagents.find((row) => row.id === child.taskId);
4954 assert.equal(task?.status, "cancelled");
4955 assert.equal(task?.completionDelivery?.state, "claimed");
4956
4957 // The parent's Stop committed, but the restart cut the effect that stops its children.
4958 const stoppedChild = yield* seedRestartCancelledChild({
4959 parentThreadId: threadId,
4960 projectId,
4961 parentRunId: runId,
4962 rootNodeId,
4963 name: "restart-parent-stopped-child",
4964 completionWake: "always",
4965 continuationPending: true,
4966 now,
4967 });
4968 yield* (yield* EventSink.EventSinkV2).write({
4969 commandId: CommandId.make("command:restart-declined-parent:stop"),
4970 events: [
4971 {
4972 id: EventId.make("event:restart-declined-parent:interrupt-request"),
4973 type: "turn-item.updated",
4974 threadId,
4975 runId,
4976 nodeId: rootNodeId,
4977 providerInstanceId: modelSelection.instanceId,
4978 occurredAt: now,
4979 payload: {
4980 id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"),
4981 type: "run_interrupt_request",
4982 threadId,
4983 runId,
4984 nodeId: rootNodeId,
4985 providerThreadId: null,
4986 providerTurnId: null,
4987 nativeItemRef: null,
4988 parentItemId: null,
4989 ordinal: 90,
4990 status: "completed",
4991 title: "Interrupt requested",
4992 startedAt: now,
4993 completedAt: now,
4994 updatedAt: now,
4995 message: "Stopped by user",
4996 },
4997 },
4998 ],
4999 });
5000 yield* continueRestartedRun({
Showing first 5,000 of 5,965 lines. View raw