@devries-cameron20 / t3code Mirror · read-only / commits / 7462f234e8
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 21:00 UTC

fix(server): preserve delegated results across wake turns (#34568)

Yash Singh committed Nov 11, 2026 at 21:01 UTC 7462f234e89027943cf0d1a73d3ae1f36571d4e3
14 files changed +669 -157
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+185 -13
@@ -10827,7 +10827,7 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10827 );
10828
10829 it.effect(
10830 - "uses task_started as an incremental roster fallback and clears on empty snapshot",
10830 + "uses task_started as an incremental roster fallback and clears internal work on empty snapshot",
10831 () =>
10832 Effect.scoped(
10833 Effect.gen(function* () {
@@ -10945,13 +10945,13 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10945 ),
10946 );
10947
10948 - yield* Queue.offer(harness.sdkMessages, emptyRoster);
10949 - yield* harness.until(() =>
10950 - providerThreadRosterEvents(harness.events).some(
10951 - (event) =>
10952 - event.providerThread.status === "idle" &&
10953 - (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0,
10954 - ),
10948 + yield* harness.offerAndWait(emptyRoster);
10949 + assert.include(
10950 + (
10951 + providerThreadRosterEvents(harness.events).at(-1)?.providerThread
10952 + .pendingBackgroundTasks ?? []
10953 + ).map((task) => task.taskId),
10954 + WAKE_TASK_ID,
10955 );
10956 assert.isFalse(yield* harness.hasPendingBackgroundWork);
10957 }).pipe(
@@ -11029,9 +11029,6 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11029 ["dev-server", false],
11030 ],
11031 );
11032 - yield* Queue.offer(harness.sdkMessages, turnOneResult);
11033 - yield* Queue.take(harness.terminalReceipts);
11034 -
11032 // A replacing level rebuilds every entry; the mark must survive it.
11033 yield* Queue.offer(
11034 harness.sdkMessages,
@@ -11045,10 +11042,10 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11042 ],
11043 );
11044
11048 - // Once the task leaves the roster, a reused id starts unmarked.
11045 + // Once the task ends, a reused id starts unmarked.
11046 yield* Queue.offer(
11047 harness.sdkMessages,
11051 - roster(["dev-server"], "00000000-0000-4000-8000-000000000303"),
11048 + claudeSdkFrame({ ...wakeNotification, uuid: "00000000-0000-4000-8000-000000000303" }),
11049 );
11050 yield* nextRoster((tasks) => tasks.length === 1);
11051 yield* Queue.offer(
@@ -11085,6 +11082,8 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11082 [WAKE_TASK_ID, false],
11083 ],
11084 );
11085 + yield* Queue.offer(harness.sdkMessages, turnOneResult);
11086 + yield* Queue.take(harness.terminalReceipts);
11087 }).pipe(
11088 Effect.provide(
11089 Layer.mergeAll(
@@ -13882,6 +13881,13 @@ describe("ClaudeAdapterV2 background wake turns", () => {
13881 const failed = harness.continuationRequests[0]!;
13882 assert.isDefined(failed.clearIfCurrent);
13883 yield* failed.clearIfCurrent!();
13884 + yield* harness.until(
13885 + () =>
13886 + !(
13887 + providerThreadRosterEvents(harness.events).at(-1)?.providerThread
13888 + .pendingBackgroundTasks ?? []
13889 + ).some((task) => task.taskId === WAKE_TASK_ID),
13890 + );
13891
13892 yield* harness.offerAndWait(
13893 claudeSdkFrame({
@@ -14605,6 +14611,172 @@ describe("ClaudeAdapterV2 background wake turns", () => {
14611 ),
14612 );
14613
14614 + it.effect.each(["none", "before", "after", "sibling"] as const)(
14615 + "keeps an exited background command on the roster until its wake run starts (empty level %s notification)",
14616 + (emptyLevel) =>
14617 + Effect.scoped(
14618 + Effect.gen(function* () {
14619 + const harness = yield* makeWakeHarness;
14620 + const now = yield* DateTime.now;
14621 + const secondTaskId = "second-owed-wake";
14622 + const taskIds = emptyLevel === "sibling" ? [WAKE_TASK_ID, secondTaskId] : [WAKE_TASK_ID];
14623 + const wakeAttemptId = RunAttemptId.make("attempt-claude-wake-roster-2");
14624 + const emptyRoster = claudeSdkFrame({
14625 + type: "system",
14626 + subtype: "background_tasks_changed",
14627 + tasks: [],
14628 + uuid: "00000000-0000-4000-8000-000000000204",
14629 + session_id: WAKE_NATIVE_SESSION,
14630 + });
14631 +
14632 + yield* harness.runtime.startTurn(
14633 + makeClaudeTestTurnInput({
14634 + threadId: harness.threadId,
14635 + providerThread: harness.providerThread,
14636 + now,
14637 + attemptId: RunAttemptId.make("attempt-claude-wake-roster-1"),
14638 + text: "Run the build in the background.",
14639 + attachments: [],
14640 + }),
14641 + );
14642 + yield* harness.offerAndWait(wakeTaskStarted);
14643 + if (emptyLevel === "sibling")
14644 + yield* harness.offerAndWait(
14645 + claudeSdkFrame({
14646 + ...wakeTaskStarted,
14647 + task_id: secondTaskId,
14648 + tool_use_id: "toolu_second_owed_wake",
14649 + uuid: "00000000-0000-4000-8000-000000000205",
14650 + }),
14651 + );
14652 + const awaitBackgroundTasks = harness.runtime.awaitBackgroundTasks;
14653 + assert.isDefined(awaitBackgroundTasks);
14654 + yield* awaitBackgroundTasks!({
14655 + providerThread: harness.providerThread,
14656 + taskIds,
14657 + awaited: true,
14658 + });
14659 + yield* Queue.offer(harness.sdkMessages, turnOneResult);
14660 + yield* harness.awaitTerminal(1);
14661 + const afterFirstTurn = harness.events.length;
14662 + if (emptyLevel === "before" || emptyLevel === "sibling")
14663 + yield* harness.offerAndWait(emptyRoster);
14664 + yield* harness.offerAndWait(wakeNotification);
14665 + if (emptyLevel === "after") yield* harness.offerAndWait(emptyRoster);
14666 + // Query replacement reads the internal roster, which the exit already cleared.
14667 + assert.isFalse(yield* harness.hasPendingBackgroundWork);
14668 + yield* harness.offerAndWait(wakeTurnInit);
14669 + yield* harness.runtime.startTurn(
14670 + makeClaudeTestTurnInput({
14671 + threadId: harness.threadId,
14672 + providerThread: harness.providerThread,
14673 + now,
14674 + attemptId: wakeAttemptId,
14675 + text: "Background task completed.",
14676 + attachments: [],
14677 + providerTurnOrdinal: 2,
14678 + messageCreatedBy: "agent",
14679 + messageCreationSource: "provider",
14680 + }),
14681 + );
14682 + yield* harness.offerAndWait(wakeAssistant);
14683 + yield* harness.offerAndWait(wakeResult);
14684 + yield* harness.awaitTerminal(2);
14685 +
14686 + const wakeStart = harness.events.findIndex(
14687 + (event) =>
14688 + event.type === "provider_turn.updated" &&
14689 + event.providerTurn.runAttemptId === wakeAttemptId,
14690 + );
14691 + assert.isAbove(wakeStart, -1);
14692 + const rosters = harness.events.flatMap((event, index) =>
14693 + event.type === "provider_thread.updated" &&
14694 + event.providerThread.id === harness.providerThread.id &&
14695 + index >= afterFirstTurn
14696 + ? [
14697 + {
14698 + beforeWake: index < wakeStart,
14699 + pendingWork: event.pendingBackgroundWork === true,
14700 + taskIds: (event.providerThread.pendingBackgroundTasks ?? []).map((task) => [
14701 + task.taskId,
14702 + task.awaited === true,
14703 + ]),
14704 + },
14705 + ]
14706 + : [],
14707 + );
14708 + assert.deepEqual(
14709 + rosters.filter((roster) => roster.beforeWake).map((roster) => roster.taskIds),
14710 + rosters
14711 + .filter((roster) => roster.beforeWake)
14712 + .map(() => taskIds.map((id) => [id, true])),
14713 + );
14714 + assert.isTrue(
14715 + rosters.filter((roster) => roster.beforeWake).every((roster) => roster.pendingWork),
14716 + );
14717 + assert.deepEqual(
14718 + rosters.at(-1)?.taskIds,
14719 + emptyLevel === "sibling" ? [[secondTaskId, true]] : [],
14720 + );
14721 + if (emptyLevel === "sibling") {
14722 + assert.isTrue(rosters.at(-1)?.pendingWork);
14723 + yield* harness.offerAndWait(
14724 + claudeSdkFrame({
14725 + ...wakeNotification,
14726 + task_id: secondTaskId,
14727 + tool_use_id: "toolu_second_owed_wake",
14728 + uuid: "00000000-0000-4000-8000-000000000206",
14729 + }),
14730 + );
14731 + yield* harness.offerAndWait(
14732 + claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000207" }),
14733 + );
14734 + yield* harness.runtime.startTurn(
14735 + makeClaudeTestTurnInput({
14736 + threadId: harness.threadId,
14737 + providerThread: harness.providerThread,
14738 + now,
14739 + attemptId: RunAttemptId.make("attempt-claude-owed-sibling-3"),
14740 + text: "Second command completed.",
14741 + attachments: [],
14742 + providerTurnOrdinal: 3,
14743 + messageCreatedBy: "agent",
14744 + messageCreationSource: "provider",
14745 + }),
14746 + );
14747 + yield* harness.offerAndWait(
14748 + makeAssistantTextFrame({
14749 + uuid: "00000000-0000-4000-8000-000000000208",
14750 + text: "Second command finished.",
14751 + }),
14752 + );
14753 + yield* harness.offerAndWait(
14754 + makeResultFrame({
14755 + uuid: "00000000-0000-4000-8000-000000000209",
14756 + result: "Second command finished.",
14757 + origin: { kind: "task-notification" },
14758 + }),
14759 + );
14760 + yield* harness.awaitTerminal(3);
14761 + assert.deepEqual(
14762 + providerThreadRosterEvents(harness.events).at(-1)?.providerThread
14763 + .pendingBackgroundTasks,
14764 + [],
14765 + );
14766 + }
14767 + }).pipe(
14768 + Effect.provide(
14769 + Layer.mergeAll(
14770 + IdAllocator.layer,
14771 + McpProviderSessions.layer,
14772 + NodeServices.layer,
14773 + NodeCrypto.layer,
14774 + ),
14775 + ),
14776 + ),
14777 + ),
14778 + );
14779 +
14780 it.effect.each([true, false])(
14781 "carries a rejected wake rate limit into the continuation failure (event=%s)",
14782 (standaloneEvent) =>
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+71 -12
@@ -6040,6 +6040,24 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6040 taskId,
6041 );
6042
6043 + // Untraced with roster emission: this runs for each streamed roster update.
6044 + const owedWakeTasksForNativeThread = Effect.fnUntraced(function* (nativeThreadId: string) {
6045 + const previous = (yield* Ref.get(lastProviderThreadByNativeThread)).get(nativeThreadId);
6046 + const roster = rosterForNativeThread(
6047 + yield* Ref.get(pendingBackgroundTasksByNativeThread),
6048 + nativeThreadId,
6049 + );
6050 + const tasks: OrchestrationV2PendingBackgroundTask[] = [];
6051 + for (const task of previous?.pendingBackgroundTasks ?? []) {
6052 + if (
6053 + !roster.has(task.taskId) &&
6054 + (yield* isKnownOpaqueBackgroundTaskOnNativeThread(nativeThreadId, task.taskId))
6055 + )
6056 + tasks.push(task);
6057 + }
6058 + return tasks;
6059 + });
6060 +
6061 const emitProviderThreadRoster = Effect.fnUntraced(function* (input: {
6062 readonly nativeThreadId: string;
6063 readonly providerThread: OrchestrationV2ProviderThread;
@@ -6053,16 +6071,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6071 yield* Ref.get(awaitedBackgroundTasksByNativeThread),
6072 input.nativeThreadId,
6073 );
6074 + const owedWakeTasks = yield* owedWakeTasksForNativeThread(input.nativeThreadId);
6075 const now = yield* DateTime.now;
6076 const providerThread: OrchestrationV2ProviderThread = {
6077 ...input.providerThread,
6078 providerSessionId: session.id,
6079 ...(input.status === undefined ? {} : { status: input.status }),
6061 - pendingBackgroundTasks: publishedPendingBackgroundTasks(
6062 - input.nativeThreadId,
6063 - roster,
6064 - awaitedTaskIds,
6065 - ),
6080 + pendingBackgroundTasks: [
6081 + ...publishedPendingBackgroundTasks(input.nativeThreadId, roster, awaitedTaskIds),
6082 + ...owedWakeTasks,
6083 + ],
6084 ...(goalsByNativeThread.has(input.nativeThreadId)
6085 ? { goal: goalsByNativeThread.get(input.nativeThreadId) ?? null }
6086 : {}),
@@ -6073,6 +6091,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6091 type: "provider_thread.updated",
6092 driver: CLAUDE_PROVIDER,
6093 providerThread,
6094 + pendingBackgroundWork: owedWakeTasks.length > 0,
6095 });
6096 });
6097
@@ -8665,6 +8684,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8684 yield* Ref.get(awaitedBackgroundTasksByNativeThread),
8685 nativeThreadId,
8686 );
8687 + const owedWakeTasks =
8688 + nativeThreadId === null
8689 + ? []
8690 + : yield* owedWakeTasksForNativeThread(nativeThreadId);
8691 // A stopped turn that wrote to the transcript moved past the pinned head.
8692 const clearConversationHead =
8693 (input.status === "completed" || input.context.nativeMessageCursor !== null) &&
@@ -8703,11 +8726,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8726 input.context.input.providerThread.firstRunOrdinal ??
8727 input.context.input.runOrdinal,
8728 lastRunOrdinal: input.context.input.runOrdinal,
8706 - pendingBackgroundTasks: publishedPendingBackgroundTasks(
8707 - nativeThreadId,
8708 - roster,
8709 - awaitedTaskIds,
8710 - ),
8729 + pendingBackgroundTasks: [
8730 + ...publishedPendingBackgroundTasks(nativeThreadId, roster, awaitedTaskIds),
8731 + ...owedWakeTasks,
8732 + ],
8733 status: input.status === "completed" ? "active" : "idle",
8734 updatedAt: input.completedAt,
8735 };
@@ -8716,6 +8738,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8738 type: "provider_thread.updated" as const,
8739 driver: CLAUDE_PROVIDER,
8740 providerThread,
8741 + pendingBackgroundWork: owedWakeTasks.length > 0,
8742 });
8743 }),
8744 emitProviderEvent(terminalEvent),
@@ -9235,6 +9258,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9258 ...later,
9259 ];
9260 if (!(yield* replaceWakeBuffer(nativeThreadId, buffered, kept, retiredTaskIds))) return;
9261 + for (const taskId of retiredTaskIds) {
9262 + yield* clearPendingBackgroundTask(nativeThreadId, taskId);
9263 + yield* clearTaskIdFromNativeThreadSet(
9264 + wakeEligibleBackgroundTasksByNativeThread,
9265 + nativeThreadId,
9266 + taskId,
9267 + );
9268 + yield* clearOpaqueBackgroundTaskReplayTombstone(nativeThreadId, taskId);
9269 + }
9270 + yield* endClaudeMonitorTasks((taskId) => retiredTaskIds.has(taskId));
9271 + const providerThread = (yield* Ref.get(lastProviderThreadByNativeThread)).get(
9272 + nativeThreadId,
9273 + );
9274 + if (providerThread !== undefined)
9275 + yield* emitProviderThreadRoster({ nativeThreadId, providerThread });
9276 if (latch.end === undefined) {
9277 // A lifecycle result closes the root frames before it without opening the wake's turn.
9278 const sinceLifecycleResult = retired.slice(
@@ -9355,7 +9393,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9393 : baseThread.status === "idle"
9394 ? ("active" as const)
9395 : baseThread.status;
9358 - yield* emitProviderThreadRoster({ nativeThreadId, providerThread: baseThread, status });
9396 + yield* emitProviderThreadRoster({
9397 + nativeThreadId,
9398 + providerThread: baseThread,
9399 + status,
9400 + });
9401 });
9402
9403 const applyBackgroundTaskRosterMessage = Effect.fnUntraced(function* (input: {
@@ -11961,7 +12003,24 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
12003 });
12004 return updated;
12005 });
11964 - yield* rememberProviderThread(turnInput.providerThread);
12006 + // A queued turn's snapshot can predate another task's exit.
12007 + const owedWakeTasks = yield* owedWakeTasksForNativeThread(nativeThreadId);
12008 + yield* rememberProviderThread(
12009 + owedWakeTasks.length === 0
12010 + ? turnInput.providerThread
12011 + : {
12012 + ...turnInput.providerThread,
12013 + pendingBackgroundTasks: [
12014 + ...(turnInput.providerThread.pendingBackgroundTasks ?? []),
12015 + ...owedWakeTasks.filter(
12016 + (task) =>
12017 + !(turnInput.providerThread.pendingBackgroundTasks ?? []).some(
12018 + (current) => current.taskId === task.taskId,
12019 + ),
12020 + ),
12021 + ],
12022 + },
12023 + );
12024 if (!goalsByNativeThread.has(nativeThreadId)) {
12025 goalsByNativeThread.set(nativeThreadId, turnInput.providerThread.goal ?? null);
12026 }
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+314 -102
@@ -1,3 +1,6 @@
1 +import * as Exit from "effect/Exit";
2 +import * as EffectOutbox from "./EffectOutbox.ts";
3 +import * as EffectWorker from "./EffectWorker.ts";
4 import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
5 import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
6 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -118,6 +121,7 @@ const layerTest = Layer.mergeAll(
121 RuntimeLayer.layer,
122 RuntimeLayer.layerEventSink,
123 ThreadCommandExecutor.layer,
124 + EffectOutbox.layer,
125 ).pipe(
126 Layer.provideMerge(RuntimeLayer.layerProjectService),
127 Layer.provide(
@@ -2428,138 +2432,344 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
2432 }),
2433 );
2434
2431 - it.effect("delivers the run a background command's wake starts after the original result", () =>
2435 + it.effect("checkpoint retry releases a pending sibling after its wake offer failed", () =>
2436 Effect.gen(function* () {
2437 const orchestrator = yield* Orchestrator.OrchestratorV2;
2434 - const eventSink = yield* EventSink.EventSinkV2;
2438 + const sink = yield* EventSink.EventSinkV2;
2439 + const outbox = yield* EffectOutbox.EffectOutboxV2;
2440 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2441 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2442 const now = yield* DateTime.now;
2437 - const parentThreadId = ThreadId.make("thread:background-wake-parent");
2438 - const projectId = ProjectId.make("project:background-wake");
2439 - const parentRunId = RunId.make("run:background-wake-parent");
2440 - const rootNodeId = NodeId.make("node:background-wake-parent");
2443 + const threadId = ThreadId.make("thread:checkpoint-own-delivery");
2444 + const originRunId = RunId.make("run:checkpoint-own-delivery:origin");
2445 + const firstId = NodeId.make("node:checkpoint-own-delivery:first");
2446 + const siblingId = NodeId.make("node:checkpoint-own-delivery:sibling");
2447 yield* seedParentWithTerminalTask({
2442 - threadId: parentThreadId,
2443 - projectId,
2444 - runId: parentRunId,
2445 - rootNodeId,
2446 - taskId: NodeId.make("node:background-wake-seed"),
2447 - deliveryState: "acknowledged",
2448 - now,
2449 - });
2450 - const child = yield* seedRestartCancelledChild({
2451 - parentThreadId,
2452 - projectId,
2453 - parentRunId,
2454 - rootNodeId,
2455 - name: "background-wake-child",
2456 - completionWake: "always",
2457 - continuationPending: false,
2458 - runStatus: "completed",
2448 + threadId,
2449 + projectId: ProjectId.make("project:checkpoint-own-delivery"),
2450 + runId: originRunId,
2451 + rootNodeId: NodeId.make("node:checkpoint-own-delivery:root"),
2452 + taskId: firstId,
2453 + deliveryState: "claimed",
2454 + deliveryTaskIds: [firstId],
2455 + parentRunStatus: "completed",
2456 now,
2457 });
2461 - yield* orchestrator.recoverDelegatedTasks;
2462 - let firstWake = yield* requests.take;
2463 - while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
2464 - yield* orchestrator.dispatch({
2465 - type: "delegated_task.completion-delivery.acknowledge",
2466 - commandId: CommandId.make("command:background-wake-ack"),
2467 - parentThreadId,
2468 - taskId: child.taskId,
2469 - observedByRunId: parentRunId,
2470 - });
2471 - const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2472 - (run) => run.id === parentRunId,
2473 - )!;
2474 - yield* eventSink.write({
2475 - commandId: reconcileCommandId("background-wake-parent-finished"),
2476 - events: [
2477 - {
2478 - id: EventId.make("event:background-wake-parent-finished"),
2479 - type: "run.updated",
2480 - threadId: parentThreadId,
2481 - runId: parentRunId,
2482 - occurredAt: now,
2483 - payload: { ...parentRun, status: "completed", completedAt: now },
2484 - },
2485 - ],
2486 - });
2487 -
2488 - // The shape ProviderContinuationService dispatches when the provider
2489 - // reports a background command (wire kind `background_command`) ended.
2490 - const wakeMessageId = MessageId.make("message:background-wake");
2458 + const original = yield* orchestrator.getThreadProjection(threadId);
2459 + const first = original.subagents.find((task) => task.id === firstId)!;
2460 + const delivery = original.runs.find((run) => run.id === originRunId)!.delegatedCompletion!
2461 + .delivery!;
2462 yield* orchestrator.dispatch({
2463 type: "message.dispatch",
2464 createdBy: "agent",
2494 - creationSource: "provider",
2495 - commandId: CommandId.make("command:background-wake"),
2496 - threadId: child.childThreadId,
2497 - messageId: wakeMessageId,
2498 - text: "Background activity updated",
2499 - notification: {
2500 - source: { kind: "command" },
2501 - outcome: "completed",
2502 - summary: 'Command "sleep 60" finished',
2503 - },
2465 + creationSource: "server",
2466 + commandId: reconcileCommandId("checkpoint-own-delivery-wake"),
2467 + threadId,
2468 + messageId: delivery.messageId,
2469 + text: "First result",
2470 attachments: [],
2471 dispatchMode: { type: "queue_after_active" },
2472 + delegatedCompletion: {
2473 + parentRunId: originRunId,
2474 + generation: delivery.generation,
2475 + taskIds: delivery.taskIds,
2476 + },
2477 });
2507 - const run = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
2508 - (row) => row.userMessageId === wakeMessageId,
2478 + const started = yield* orchestrator.getThreadProjection(threadId);
2479 + const run = started.runs.find(
2480 + (candidate) => candidate.userMessageId === delivery.messageId,
2481 )!;
2510 - assert.isDefined(run.delegatedTaskId);
2511 - yield* eventSink.write({
2512 - commandId: reconcileCommandId("background-wake-finished"),
2482 + const scopeId = started.nodes.find(
2483 + (node) => node.id === run.rootNodeId,
2484 + )!.checkpointScopeId!;
2485 + yield* outbox.cancelUnsettled({
2486 + threadId,
2487 + effectTypes: ["provider-turn.start"],
2488 + reason: "The delivery turn already completed.",
2489 + });
2490 + const commandId = reconcileCommandId("checkpoint-own-delivery-finished");
2491 + yield* sink.writeWithEffects({
2492 + commandId,
2493 events: [
2494 {
2515 - id: EventId.make("event:background-wake-result"),
2516 - type: "message.updated",
2517 - threadId: child.childThreadId,
2518 - runId: run.id,
2495 + id: EventId.make("event:checkpoint-own-delivery:sibling"),
2496 + type: "subagent.updated",
2497 + threadId,
2498 + runId: originRunId,
2499 + nodeId: siblingId,
2500 occurredAt: now,
2501 payload: {
2521 - id: MessageId.make("message:background-wake-result"),
2522 - threadId: child.childThreadId,
2523 - runId: run.id,
2524 - nodeId: null,
2525 - role: "assistant",
2526 - text: "The background command finished cleanly.",
2527 - attachments: [],
2528 - streaming: false,
2529 - createdBy: "agent",
2530 - creationSource: "server",
2531 - createdAt: now,
2532 - updatedAt: now,
2502 + ...first,
2503 + id: siblingId,
2504 + completionDelivery: { state: "pending", observedByRunId: null },
2505 },
2506 },
2507 {
2536 - id: EventId.make("event:background-wake-finished"),
2508 + id: EventId.make("event:checkpoint-own-delivery:finished"),
2509 type: "run.updated",
2538 - threadId: child.childThreadId,
2510 + threadId,
2511 runId: run.id,
2512 occurredAt: now,
2541 - payload: { ...run, status: "completed", startedAt: now, completedAt: now },
2513 + payload: {
2514 + ...run,
2515 + status: "completed",
2516 + startedAt: now,
2517 + completedAt: now,
2518 + checkpointId: CheckpointId.make("checkpoint:own-delivery"),
2519 + },
2520 + },
2521 + ],
2522 + effects: [
2523 + {
2524 + id: "effect:checkpoint-own-delivery",
2525 + commandId,
2526 + threadId,
2527 + request: { type: "checkpoint.capture", runId: run.id, scopeId },
2528 },
2529 ],
2530 });
2545 - yield* orchestrator.recoverDelegatedTasks;
2546 - const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2547 - (task) => task.id === run.delegatedTaskId,
2548 - )!;
2549 - assert.equal(followup.status, "completed");
2550 - assert.equal(followup.result, "The background command finished cleanly.");
2551 - assert.equal(followup.completionDelivery?.state, "claimed");
2531 + using offer = vi
2532 + .spyOn(requests, "offer")
2533 + .mockImplementationOnce(() => Effect.die("successor wake offer failed"));
2534 + const firstAttempt = yield* Effect.exit(
2535 + orchestrator.recoverDelegatedTask(threadId, { completedRunId: run.id }),
2536 + );
2537 + assert.isTrue(Exit.isFailure(firstAttempt));
2538 + const reserved = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
2539 + (candidate) => candidate.id === originRunId,
2540 + )!.delegatedCompletion!.delivery!;
2541 + assert.deepEqual(reserved.taskIds, [siblingId]);
2542 + yield* worker.drain();
2543 + assert.lengthOf(offer.mock.calls, 2);
2544 + const completed = yield* orchestrator.getThreadProjection(threadId);
2545 + assert.equal(
2546 + completed.subagents.find((task) => task.id === firstId)?.completionDelivery?.state,
2547 + "delivered",
2548 + );
2549 + assert.equal(
2550 + completed.subagents.find((task) => task.id === siblingId)?.completionDelivery?.state,
2551 + "claimed",
2552 + );
2553 + assert.deepEqual(
2554 + completed.runs.find((candidate) => candidate.id === originRunId)?.delegatedCompletion
2555 + ?.delivery?.taskIds,
2556 + [siblingId],
2557 + );
2558 let wake = yield* requests.take;
2553 - while (
2554 - wake.threadId !== parentThreadId ||
2555 - wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId
2556 - ) {
2557 - wake = yield* requests.take;
2558 - }
2559 - assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId);
2559 + while (wake.threadId !== threadId) wake = yield* requests.take;
2560 + assert.equal(wake.delegatedCompletion?.generation, delivery.generation + 1);
2561 }),
2562 );
2563
2564 + it.effect.each(["live", "checkpoint-retry", "checkpoint-offer-retry"] as const)(
2565 + "delivers the run a background command's wake starts after the original result (%s)",
2566 + (delivery) =>
2567 + Effect.gen(function* () {
2568 + const orchestrator = yield* Orchestrator.OrchestratorV2;
2569 + const eventSink = yield* EventSink.EventSinkV2;
2570 + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2571 + const now = yield* DateTime.now;
2572 + const parentThreadId = ThreadId.make(`thread:background-wake-parent-${delivery}`);
2573 + const projectId = ProjectId.make(`project:background-wake-${delivery}`);
2574 + const parentRunId = RunId.make(`run:background-wake-parent-${delivery}`);
2575 + const rootNodeId = NodeId.make(`node:background-wake-parent-${delivery}`);
2576 + yield* seedParentWithTerminalTask({
2577 + threadId: parentThreadId,
2578 + projectId,
2579 + runId: parentRunId,
2580 + rootNodeId,
2581 + taskId: NodeId.make(`node:background-wake-seed-${delivery}`),
2582 + deliveryState: "acknowledged",
2583 + now,
2584 + });
2585 + const child = yield* seedRestartCancelledChild({
2586 + parentThreadId,
2587 + projectId,
2588 + parentRunId,
2589 + rootNodeId,
2590 + name: `background-wake-child-${delivery}`,
2591 + completionWake: "always",
2592 + continuationPending: false,
2593 + runStatus: "completed",
2594 + now,
2595 + });
2596 + yield* orchestrator.recoverDelegatedTasks;
2597 + let firstWake = yield* requests.take;
2598 + while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
2599 + yield* orchestrator.dispatch({
2600 + type: "delegated_task.completion-delivery.acknowledge",
2601 + commandId: CommandId.make(`command:background-wake-ack-${delivery}`),
2602 + parentThreadId,
2603 + taskId: child.taskId,
2604 + observedByRunId: parentRunId,
2605 + });
2606 + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2607 + (run) => run.id === parentRunId,
2608 + )!;
2609 + yield* eventSink.write({
2610 + commandId: reconcileCommandId(`background-wake-parent-finished-${delivery}`),
2611 + events: [
2612 + {
2613 + id: EventId.make(`event:background-wake-parent-finished-${delivery}`),
2614 + type: "run.updated",
2615 + threadId: parentThreadId,
2616 + runId: parentRunId,
2617 + occurredAt: now,
2618 + payload: { ...parentRun, status: "completed", completedAt: now },
2619 + },
2620 + ],
2621 + });
2622 +
2623 + // The shape ProviderContinuationService dispatches when the provider
2624 + // reports a background command (wire kind `background_command`) ended.
2625 + const wakeMessageId = MessageId.make(`message:background-wake-${delivery}`);
2626 + yield* orchestrator.dispatch({
2627 + type: "message.dispatch",
2628 + createdBy: "agent",
2629 + creationSource: "provider",
2630 + commandId:
2631 + delivery === "live"
2632 + ? CommandId.make(`command:background-wake-${delivery}`)
2633 + : reconcileCommandId(`background-wake-${delivery}`),
2634 + threadId: child.childThreadId,
2635 + messageId: wakeMessageId,
2636 + text: "Background activity updated",
2637 + notification: {
2638 + source: { kind: "command" },
2639 + outcome: "completed",
2640 + summary: 'Command "sleep 60" finished',
2641 + },
2642 + attachments: [],
2643 + dispatchMode: { type: "queue_after_active" },
2644 + });
2645 + const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId);
2646 + const run = childProjection.runs.find((row) => row.userMessageId === wakeMessageId)!;
2647 + const scopeId = childProjection.nodes.find(
2648 + (node) => node.id === run.rootNodeId,
2649 + )!.checkpointScopeId!;
2650 + const commandId =
2651 + delivery === "live"
2652 + ? CommandId.make(`command:background-wake-finished-${delivery}`)
2653 + : reconcileCommandId(`background-wake-finished-${delivery}`);
2654 + if (delivery !== "live") {
2655 + const outbox = yield* EffectOutbox.EffectOutboxV2;
2656 + yield* outbox.cancelUnsettled({
2657 + threadId: child.childThreadId,
2658 + effectTypes: ["provider-turn.start"],
2659 + reason: "The turn already finished before its checkpoint retry.",
2660 + });
2661 + }
2662 + const afterSequence = yield* eventSink.latestSequence();
2663 + assert.isDefined(run.delegatedTaskId);
2664 + yield* eventSink.writeWithEffects({
2665 + commandId,
2666 + effects:
2667 + delivery === "live"
2668 + ? []
2669 + : [
2670 + {
2671 + id: `effect:background-wake-checkpoint-retry-${delivery}`,
2672 + commandId,
2673 + threadId: child.childThreadId,
2674 + request: { type: "checkpoint.capture", runId: run.id, scopeId },
2675 + },
2676 + ],
2677 + events: [
2678 + {
2679 + id: EventId.make(`event:background-wake-result-${delivery}`),
2680 + type: "message.updated",
2681 + threadId: child.childThreadId,
2682 + runId: run.id,
2683 + occurredAt: now,
2684 + payload: {
2685 + id: MessageId.make(`message:background-wake-result-${delivery}`),
2686 + threadId: child.childThreadId,
2687 + runId: run.id,
2688 + nodeId: null,
2689 + role: "assistant",
2690 + text: "The background command finished cleanly.",
2691 + attachments: [],
2692 + streaming: false,
2693 + createdBy: "agent",
2694 + creationSource: "server",
2695 + createdAt: now,
2696 + updatedAt: now,
2697 + },
2698 + },
2699 + {
2700 + id: EventId.make(`event:background-wake-finished-${delivery}`),
2701 + type: "run.updated",
2702 + threadId: child.childThreadId,
2703 + runId: run.id,
2704 + occurredAt: now,
2705 + payload: {
2706 + ...run,
2707 + status: "completed",
2708 + startedAt: now,
2709 + completedAt: now,
2710 + checkpointId:
2711 + delivery === "live"
2712 + ? null
2713 + : CheckpointId.make(`checkpoint:background-wake-${delivery}`),
2714 + },
2715 + },
2716 + ],
2717 + });
2718 + if (delivery !== "live") {
2719 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2720 + if (delivery === "checkpoint-offer-retry") {
2721 + using offer = vi
2722 + .spyOn(requests, "offer")
2723 + .mockImplementationOnce(() => Effect.die("wake offer failed"));
2724 + const first = yield* Effect.exit(
2725 + orchestrator.recoverDelegatedTask(child.childThreadId),
2726 + );
2727 + assert.isTrue(Exit.isFailure(first));
2728 + const committed = (yield* orchestrator.getThreadProjection(
2729 + parentThreadId,
2730 + )).subagents.find((task) => task.id === run.delegatedTaskId)!;
2731 + assert.equal(committed.result, "The background command finished cleanly.");
2732 + yield* worker.drain();
2733 + assert.lengthOf(offer.mock.calls, 2);
2734 + } else {
2735 + yield* worker.drain();
2736 + }
2737 + } else {
2738 + yield* eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
2739 + Stream.filter(
2740 + (stored) =>
2741 + stored.event.type === "subagent.updated" &&
2742 + stored.event.payload.id === run.delegatedTaskId &&
2743 + stored.event.payload.status === "completed",
2744 + ),
2745 + Stream.take(1),
2746 + Stream.runDrain,
2747 + );
2748 + }
2749 + const completed = yield* orchestrator.getThreadProjection(parentThreadId);
2750 + const followup = completed.subagents.find((task) => task.id === run.delegatedTaskId)!;
2751 + assert.lengthOf(
2752 + completed.contextTransfers.filter(
2753 + (transfer) =>
2754 + transfer.type === "subagent_result" &&
2755 + transfer.sourceThreadId === child.childThreadId,
2756 + ),
2757 + 2,
2758 + );
2759 + assert.equal(followup.status, "completed");
2760 + assert.equal(followup.result, "The background command finished cleanly.");
2761 + assert.equal(followup.completionDelivery?.state, "claimed");
2762 + let wake = yield* requests.take;
2763 + while (
2764 + wake.threadId !== parentThreadId ||
2765 + wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId
2766 + ) {
2767 + wake = yield* requests.take;
2768 + }
2769 + assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId);
2770 + }),
2771 + );
2772 +
2773 // The child's follow-up run sends `t3_thread_send` to its own thread before it ends.
2774 const startSelfQueuedFollowup = (name: string) =>
2775 Effect.gen(function* () {
@@ -3671,7 +3881,9 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
3881 assert.isTrue(yield* orchestrator.delegatedTaskResultPending(recut.childThreadId));
3882 assert.isTrue(yield* orchestrator.delegatedTaskResultPending(resumed.childThreadId));
3883 assert.isFalse(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
3674 - yield* orchestrator.recoverDelegatedTask(recut.childThreadId, recut.childRunId);
3884 + yield* orchestrator.recoverDelegatedTask(recut.childThreadId, {
3885 + sourceRunId: recut.childRunId,
3886 + });
3887 const replayed = yield* orchestrator.getThreadProjection(threadId);
3888 assert.equal(replayed.subagents.find((row) => row.id === recut.taskId)?.status, "running");
3889 yield* eventSink.write({
apps/server/src/orchestration-v2/EffectWorker.ts
+7 -1
@@ -106,7 +106,7 @@ export const layerExecutor: Layer.Layer<
106 Effect.tapError(() =>
107 willRetry
108 ? Effect.void
109 - : threads.recoverDelegatedTask(effect.threadId, sourceRunId),
109 + : threads.recoverDelegatedTask(effect.threadId, { sourceRunId }),
110 ),
111 Effect.mapError(
112 (cause) =>
@@ -506,6 +506,12 @@ export const layerExecutor: Layer.Layer<
506 scopeId: effect.request.scopeId,
507 })
508 .pipe(
509 + // Completion bookkeeping must survive a missed live run update, including capture retries.
510 + Effect.andThen(
511 + threads.recoverDelegatedTask(effect.threadId, {
512 + completedRunId: effect.request.runId,
513 + }),
514 + ),
515 Effect.mapError(
516 (cause) =>
517 new OrchestrationEffectExecutionError({
apps/server/src/orchestration-v2/Orchestrator.ts
+40 -12
@@ -382,7 +382,10 @@ export interface OrchestratorV2DispatchResult {
382 export interface OrchestratorV2Shape {
383 readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
384 readonly recoverDelegatedTasks: Effect.Effect<void>;
385 - readonly recoverDelegatedTask: (threadId: ThreadId, sourceRunId: RunId) => Effect.Effect<void>;
385 + readonly recoverDelegatedTask: (
386 + threadId: ThreadId,
387 + options?: { readonly sourceRunId?: RunId; readonly completedRunId?: RunId },
388 + ) => Effect.Effect<void, OrchestratorProjectionError>;
389 /** Commits cancellation with the child's Stop after descendants accept stopping. */
390 readonly cancelDelegatedTask: <E, R>(input: {
391 readonly parentThreadId: ThreadId;
@@ -15054,23 +15057,48 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15057 );
15058
15059 const recoverDelegatedTask = Effect.fn("Orchestrator.recoverDelegatedTask")(
15057 - function* (threadId: ThreadId, sourceRunId: RunId) {
15060 + function* (
15061 + threadId: ThreadId,
15062 + options?: { readonly sourceRunId?: RunId; readonly completedRunId?: RunId },
15063 + ) {
15064 + const completedRunId = options?.completedRunId;
15065 + if (completedRunId !== undefined) {
15066 + yield* threadDispatch.withLock(
15067 + threadId,
15068 + Effect.gen(function* () {
15069 + const message = yield* projectionStore.getRunMessage(threadId, completedRunId);
15070 + if (message?.delegatedCompletion === undefined) return;
15071 + // A prior attempt may have committed the next generation before offering its wake.
15072 + yield* offerDelegatedCompletionDelivery(
15073 + threadId,
15074 + message.delegatedCompletion.parentRunId,
15075 + );
15076 + yield* finalizeDelegatedCompletionDelivery(threadId, completedRunId);
15077 + }),
15078 + );
15079 + }
15080 + const sourceRunId = options?.sourceRunId;
15081 const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
15082 if (parentThreadId === undefined) return;
15083 + if (
15084 + sourceRunId === undefined &&
15085 + !(yield* projectionStore.hasPendingDelegatedCompletion(threadId))
15086 + )
15087 + return;
15088 yield* threadDispatch.withLock(
15089 parentThreadId,
15062 - finalizeAppOwnedSubagent(threadId, { settledContinuationOf: sourceRunId }),
15090 + offerDelegatedCompletionDeliveries(parentThreadId).pipe(
15091 + Effect.andThen(
15092 + finalizeAppOwnedSubagent(
15093 + threadId,
15094 + sourceRunId === undefined ? undefined : { settledContinuationOf: sourceRunId },
15095 + ),
15096 + ),
15097 + ),
15098 );
15099 },
15065 - (effect, threadId, _sourceRunId) =>
15066 - effect.pipe(
15067 - Effect.catchCause((cause) =>
15068 - Effect.logWarning("Failed to recover delegated task after restart", {
15069 - childThreadId: threadId,
15070 - cause,
15071 - }),
15072 - ),
15073 - ),
15100 + (effect, threadId, _options) =>
15101 + effect.pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
15102 );
15103
15104 const cancelDelegatedTask: OrchestratorV2Shape["cancelDelegatedTask"] = Effect.fn(
apps/server/src/orchestration-v2/ProjectionStore.ts
+18
@@ -477,6 +477,9 @@ export interface ProjectionStoreV2Shape {
477 readonly listAppOwnedChildThreadIds: (
478 threadId: ThreadId,
479 ) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
480 + readonly hasPendingDelegatedCompletion: (
481 + childThreadId: ThreadId,
482 + ) => Effect.Effect<boolean, ProjectionStoreV2Error>;
483 readonly getNextTurnItemOrdinal: (
484 threadId: ThreadId,
485 ) => Effect.Effect<number, ProjectionStoreV2Error>;
@@ -5809,6 +5812,20 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5812 Effect.map((rows) => [...new Set(rows.map((row) => ThreadId.make(row.child_thread_id)))]),
5813 Effect.mapError(controlReadError(threadId)),
5814 );
5815 + const hasPendingDelegatedCompletion: ProjectionStoreV2Shape["hasPendingDelegatedCompletion"] =
5816 + Effect.fn("ProjectionStore.hasPendingDelegatedCompletion")((childThreadId) =>
5817 + sql`SELECT 1 FROM orchestration_v2_projection_subagents AS task
5818 + LEFT JOIN orchestration_v2_projection_runs AS owner ON owner.run_id = task.run_id
5819 + WHERE task.child_thread_id = ${childThreadId} AND task.origin = 'app_owned'
5820 + AND (json_extract(task.payload_json, '$.result') IS NULL
5821 + OR (task.completion_delivery_state = 'claimed'
5822 + AND json_extract(owner.payload_json, '$.delegatedCompletion.disposition') = 'open'
5823 + AND json_type(owner.payload_json, '$.delegatedCompletion.delivery') = 'object'))
5824 + LIMIT 1`.pipe(
5825 + Effect.map((rows) => rows.length > 0),
5826 + Effect.mapError(controlReadError(childThreadId)),
5827 + ),
5828 + );
5829 const getNextTurnItemOrdinal: ProjectionStoreV2Shape["getNextTurnItemOrdinal"] = (threadId) =>
5830 sql<{ ordinal: number | null }>`SELECT MAX(ordinal) AS ordinal
5831 FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId}`.pipe(
@@ -7832,6 +7849,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7849 getMessageCount: readOnReader(getMessageCount),
7850 getSubagentCount: readOnReader(getSubagentCount),
7851 listAppOwnedChildThreadIds: readOnReader(listAppOwnedChildThreadIds),
7852 + hasPendingDelegatedCompletion: readOnReader(hasPendingDelegatedCompletion),
7853 getNextTurnItemOrdinal: readOnReader(getNextTurnItemOrdinal),
7854 getTurnItem: readOnReader(getTurnItem),
7855 getMessagesById: readOnReader(getMessagesById),
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+1
@@ -239,6 +239,7 @@ it.effect(
239 getThreadHistoryPage: () => Effect.die("unused"),
240 getTimelinePage: () => Effect.die("Unused timeline read"),
241 getMessageCount: () => Effect.die("unused message count"),
242 + hasPendingDelegatedCompletion: () => Effect.die("unused hasPendingDelegatedCompletion"),
243 getSubagentCount: () => Effect.die("unused subagent count"),
244 listAppOwnedChildThreadIds: () => Effect.die("unused listAppOwnedChildThreadIds"),
245 getNextTurnItemOrdinal: () => Effect.die("unused ordinal read"),
apps/server/src/orchestration-v2/RestartContinuation.ts
+1 -1
@@ -352,7 +352,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
352 Effect.andThen(
353 Effect.gen(function* () {
354 const threads = yield* ThreadManagementService.ThreadManagementService;
355 - yield* threads.recoverDelegatedTask(input.threadId, input.sourceRunId);
355 + yield* threads.recoverDelegatedTask(input.threadId, { sourceRunId: input.sourceRunId });
356 }),
357 ),
358 ),
apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts
+3 -1
@@ -94,7 +94,9 @@ it.effect.each([
94 Layer.mock(CheckpointRollback.CheckpointRollbackServiceV2)({}),
95 Layer.mock(RuntimeRequests.RuntimeRequestServiceV2)({}),
96 Layer.mock(ThreadTitles.ThreadTitleRegenerationService)({}),
97 - Layer.mock(ThreadManagement.ThreadManagementService)({}),
97 + Layer.mock(ThreadManagement.ThreadManagementService)({
98 + recoverDelegatedTask: () => Effect.void,
99 + }),
100 ServerSettings.layerTest(),
101 Layer.mock(ProviderTurnStart.ProviderTurnStartServiceV2)({
102 start: () => Effect.sync(() => calls.push("start")).pipe(Effect.asVoid),
apps/server/src/orchestration-v2/ThreadStop.test.ts
+2 -2
@@ -1242,7 +1242,7 @@ it.effect.each([
1242 (projection) => projection.runtimeRequests[0],
1243 );
1244
1245 - yield* orchestrator.recoverDelegatedTask(childThreadId, childRun.id);
1245 + yield* orchestrator.recoverDelegatedTask(childThreadId, { sourceRunId: childRun.id });
1246 assert.equal((yield* request)?.status, "pending");
1247 if (ending === "interrupted") {
1248 yield* orchestrator.dispatch({
@@ -1260,7 +1260,7 @@ it.effect.each([
1260 payload: { ...childRun, status: ending, startedAt: now, completedAt: now },
1261 });
1262 }
1263 - yield* orchestrator.recoverDelegatedTask(childThreadId, childRun.id);
1263 + yield* orchestrator.recoverDelegatedTask(childThreadId, { sourceRunId: childRun.id });
1264 assert.equal((yield* request)?.status, question);
1265 assert.equal((yield* request)?.decision, question === "resolved" ? "cancel" : undefined);
1266 }).pipe(Effect.provide(layerTest)),
apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/claude_transcript.ndjson
+3 -1
@@ -1,4 +1,4 @@
1 -{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"claude_background_task_after_root","metadata":{"prompts":["Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."],"model":"claude-sonnet-4-6","nativeSessionId":"ec24a006-617b-4324-b516-57a7afff15a4","queryMode":"streaming","tools":"claude_code","permissionMode":"bypassPermissions","timingCompression":"25-second background gap controlled by the replay scenario gate","generatedBy":"manual-replay-from-thread-ec24a006 (ctm-catchup-l2-20260729, captured 2026-07-30T00:35:14Z)"}}
1 +{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"claude_background_task_after_root","metadata":{"prompts":["Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."],"model":"claude-sonnet-4-6","nativeSessionId":"ec24a006-617b-4324-b516-57a7afff15a4","queryMode":"streaming","tools":"claude_code","permissionMode":"bypassPermissions","timingCompression":"25-second background gap controlled by the replay scenario gate","generatedBy":"manual-replay-from-thread-ec24a006 (ctm-catchup-l2-20260729, captured 2026-07-30T00:35:14Z)","syntheticExtension":"Wake assistant and result frames extend the captured notification-only transcript to exercise completion consumption."}}
2 {"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"sessionId":"ec24a006-617b-4324-b516-57a7afff15a4","settings":{"showThinkingSummaries":true}}}}
3 {"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."},"parent_tool_use_id":null}}}
4 {"type":"emit_inbound","label":"system:init","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.219","cwd":"/tmp/claude-replay-claude_background_task_after_root","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"fdd8c2b3-f443-405c-996d-1a5ac798e100","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}}
@@ -10,4 +10,6 @@
10 {"type":"emit_inbound","label":"result:root","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":7125,"duration_api_ms":6500,"num_turns":2,"result":"L2_STARTED","stop_reason":"end_turn","session_id":"ec24a006-617b-4324-b516-57a7afff15a4","total_cost_usd":0.001,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":2,"output_tokens":9,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":2,"outputTokens":9,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"633b885b-240f-4f44-bfe1-d554088b2dfd"}}
11 {"type":"emit_inbound","label":"background_tasks_changed:empty","frame":{"type":"system","subtype":"background_tasks_changed","tasks":[],"uuid":"3464e6d2-1a25-4e37-82de-d305440ed44b","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}}
12 {"type":"emit_inbound","label":"task_notification:local_bash","frame":{"type":"system","subtype":"task_notification","task_id":"bc9gkn8ei","tool_use_id":"toolu_01DAnwwvVvLM1cTfzrm6kkor","status":"completed","output_file":"/tmp/claude-replay-claude_background_task_after_root/tasks/bc9gkn8ei.output","summary":"Background command \"sleep 25 && echo L2_BG_DONE\" completed (exit code 0)","uuid":"0a8bc186-10a1-421c-a319-e3c7db9cb4a6","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}}
13 +{"type":"emit_inbound","label":"assistant:wake-final","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_background_command_wake","type":"message","role":"assistant","content":[{"type":"text","text":"L2_WAKE: L2_BG_DONE"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"ec24a006-617b-4324-b516-57a7afff15a4","uuid":"39c3f18d-5ec3-48ab-b309-2119f5efb6c5"}}
14 +{"type":"emit_inbound","label":"result:background-wake:1","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":7125,"duration_api_ms":6500,"num_turns":1,"result":"L2_WAKE: L2_BG_DONE","stop_reason":"end_turn","session_id":"ec24a006-617b-4324-b516-57a7afff15a4","total_cost_usd":0.001,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":2,"output_tokens":9,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":2,"outputTokens":9,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"43458caa-c0da-459a-a325-af6d22bd0916","origin":{"kind":"task-notification"}}}
15 {"type":"runtime_exit","status":"success"}
apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/input.ts
+9 -3
@@ -1,3 +1,4 @@
1 +import { claudeBackgroundWakeResultLabel } from "../../../Adapters/ClaudeAdapterV2.testkit.ts";
2 import type { OrchestratorFixtureInput } from "../shared.ts";
3
4 export const CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT = [
@@ -16,11 +17,16 @@ export function claudeBackgroundTaskAfterRootInput(): OrchestratorFixtureInput {
17 return {
18 steps: [
19 { type: "message", text: CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT },
20 + { type: "await_run_status", targetRunIndex: 1, status: "completed" },
21 + { type: "release_replay_gate", label: "background_tasks_changed:empty" },
22 {
20 - type: "release_replay_gate_after_waiting",
21 - label: "background_tasks_changed:empty",
22 - targetRunIndex: 1,
23 + type: "await_run_status",
24 + targetRunIndex: 2,
25 + status: "running",
26 + waitForTurnItemType: "assistant_message",
27 },
28 + { type: "release_replay_gate", label: claudeBackgroundWakeResultLabel(1) },
29 + { type: "await_run_status", targetRunIndex: 2, status: "completed" },
30 ],
31 };
32 }
apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/output.ts
+14 -9
@@ -19,8 +19,8 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
19 assertBaseProjection({
20 result,
21 transcript,
22 - runCount: 1,
23 - runStatuses: ["completed"],
22 + runCount: 2,
23 + runStatuses: ["completed", "completed"],
24 });
25
26 const projection = projectionFor(result, transcript.scenario);
@@ -28,7 +28,9 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
28 assertUserMessagesInclude(projection, [CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT]);
29
30 const rootRun = projection.runs[0];
31 + const wakeRun = projection.runs[1];
32 assert.isDefined(rootRun);
33 + assert.isDefined(wakeRun);
34 const pendingRosterIndex = result.domainEvents.findIndex(
35 (event) =>
36 event.type === "provider-thread.updated" &&
@@ -47,12 +49,14 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
49 event.payload.kind === "root_turn" &&
50 event.payload.status === "waiting",
51 );
50 - const idleRosterIndex = result.domainEvents.findIndex(
52 + const clearedRosterIndex = result.domainEvents.findIndex(
53 (event, index) =>
54 index > waitingRunIndex &&
55 event.type === "provider-thread.updated" &&
54 - event.payload.status === "idle" &&
55 - (event.payload.pendingBackgroundTasks?.length ?? 0) === 0,
56 + event.payload.pendingBackgroundTasks?.length === 0,
57 + );
58 + const wakeRunIndex = result.domainEvents.findIndex(
59 + (event) => event.type === "run.created" && event.payload.id === wakeRun.id,
60 );
61
62 assert.isAtLeast(pendingRosterIndex, 0, "replay must project the live background task roster");
@@ -66,10 +70,11 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
70 pendingRosterIndex,
71 "checkpoint-waiting root node must retain the background roster",
72 );
73 + assert.isAbove(wakeRunIndex, waitingRunIndex, "the wake must follow the root turn");
74 assert.isAbove(
70 - idleRosterIndex,
71 - waitingRunIndex,
72 - "late background completion must clear the roster and return the provider thread to idle",
75 + clearedRosterIndex,
76 + wakeRunIndex,
77 + "the background task must remain pending until its wake run consumes the completion",
78 );
79 assert.equal(rootRun.status, "completed");
80 const rootNode = projection.nodes.find(
@@ -86,7 +91,7 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
91 const assistantTexts = projection.turnItems.flatMap((item) =>
92 item.type === "assistant_message" ? [item.text] : [],
93 );
89 - assert.deepEqual(assistantTexts, ["L2_STARTED"]);
94 + assert.deepEqual(assistantTexts, ["L2_STARTED", "L2_WAKE: L2_BG_DONE"]);
95
96 const shell = result.shellSnapshot.threads.find((thread) => thread.id === projection.thread.id);
97 assert.isDefined(shell);
apps/server/src/orchestration-v2/testkit/fixtures/index.ts
+1
@@ -287,6 +287,7 @@ export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray<OrchestratorReplayFixtu
287 import.meta.url,
288 ),
289 modelSelection: CLAUDE_MODEL_SELECTION,
290 + runContinuationWorker: true,
291 assertOutput: assertClaudeBackgroundTaskAfterRootOutput,
292 },
293 ],