The best way to control Claude Code, Codex and more from any computer or phone
fix(server): preserve delegated results across wake turns (#34568)
Yash Singh committed
Nov 11, 2026 at 21:01 UTC
7462f234e89027943cf0d1a73d3ae1f36571d4e3
14 files changed
+669
-157
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+185
-13
@@ -10827,7 +10827,7 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10827
);
10828
10829
it.effect(
10830
- "uses task_started as an incremental roster fallback and clears on empty snapshot",
10830
+ "uses task_started as an incremental roster fallback and clears internal work on empty snapshot",
10831
() =>
10832
Effect.scoped(
10833
Effect.gen(function* () {
@@ -10945,13 +10945,13 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10945
),
10946
);
10947
10948
- yield* Queue.offer(harness.sdkMessages, emptyRoster);
10949
- yield* harness.until(() =>
10950
- providerThreadRosterEvents(harness.events).some(
10951
- (event) =>
10952
- event.providerThread.status === "idle" &&
10953
- (event.providerThread.pendingBackgroundTasks?.length ?? 0) === 0,
10954
- ),
10948
+ yield* harness.offerAndWait(emptyRoster);
10949
+ assert.include(
10950
+ (
10951
+ providerThreadRosterEvents(harness.events).at(-1)?.providerThread
10952
+ .pendingBackgroundTasks ?? []
10953
+ ).map((task) => task.taskId),
10954
+ WAKE_TASK_ID,
10955
);
10956
assert.isFalse(yield* harness.hasPendingBackgroundWork);
10957
}).pipe(
@@ -11029,9 +11029,6 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11029
["dev-server", false],
11030
],
11031
);
11032
- yield* Queue.offer(harness.sdkMessages, turnOneResult);
11033
- yield* Queue.take(harness.terminalReceipts);
11034
-
11032
// A replacing level rebuilds every entry; the mark must survive it.
11033
yield* Queue.offer(
11034
harness.sdkMessages,
@@ -11045,10 +11042,10 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11042
],
11043
);
11044
11048
- // Once the task leaves the roster, a reused id starts unmarked.
11045
+ // Once the task ends, a reused id starts unmarked.
11046
yield* Queue.offer(
11047
harness.sdkMessages,
11051
- roster(["dev-server"], "00000000-0000-4000-8000-000000000303"),
11048
+ claudeSdkFrame({ ...wakeNotification, uuid: "00000000-0000-4000-8000-000000000303" }),
11049
);
11050
yield* nextRoster((tasks) => tasks.length === 1);
11051
yield* Queue.offer(
@@ -11085,6 +11082,8 @@ describe("ClaudeAdapterV2 background wake turns", () => {
11082
[WAKE_TASK_ID, false],
11083
],
11084
);
11085
+ yield* Queue.offer(harness.sdkMessages, turnOneResult);
11086
+ yield* Queue.take(harness.terminalReceipts);
11087
}).pipe(
11088
Effect.provide(
11089
Layer.mergeAll(
@@ -13882,6 +13881,13 @@ describe("ClaudeAdapterV2 background wake turns", () => {
13881
const failed = harness.continuationRequests[0]!;
13882
assert.isDefined(failed.clearIfCurrent);
13883
yield* failed.clearIfCurrent!();
13884
+ yield* harness.until(
13885
+ () =>
13886
+ !(
13887
+ providerThreadRosterEvents(harness.events).at(-1)?.providerThread
13888
+ .pendingBackgroundTasks ?? []
13889
+ ).some((task) => task.taskId === WAKE_TASK_ID),
13890
+ );
13891
13892
yield* harness.offerAndWait(
13893
claudeSdkFrame({
@@ -14605,6 +14611,172 @@ describe("ClaudeAdapterV2 background wake turns", () => {
14611
),
14612
);
14613
14614
+ it.effect.each(["none", "before", "after", "sibling"] as const)(
14615
+ "keeps an exited background command on the roster until its wake run starts (empty level %s notification)",
14616
+ (emptyLevel) =>
14617
+ Effect.scoped(
14618
+ Effect.gen(function* () {
14619
+ const harness = yield* makeWakeHarness;
14620
+ const now = yield* DateTime.now;
14621
+ const secondTaskId = "second-owed-wake";
14622
+ const taskIds = emptyLevel === "sibling" ? [WAKE_TASK_ID, secondTaskId] : [WAKE_TASK_ID];
14623
+ const wakeAttemptId = RunAttemptId.make("attempt-claude-wake-roster-2");
14624
+ const emptyRoster = claudeSdkFrame({
14625
+ type: "system",
14626
+ subtype: "background_tasks_changed",
14627
+ tasks: [],
14628
+ uuid: "00000000-0000-4000-8000-000000000204",
14629
+ session_id: WAKE_NATIVE_SESSION,
14630
+ });
14631
+
14632
+ yield* harness.runtime.startTurn(
14633
+ makeClaudeTestTurnInput({
14634
+ threadId: harness.threadId,
14635
+ providerThread: harness.providerThread,
14636
+ now,
14637
+ attemptId: RunAttemptId.make("attempt-claude-wake-roster-1"),
14638
+ text: "Run the build in the background.",
14639
+ attachments: [],
14640
+ }),
14641
+ );
14642
+ yield* harness.offerAndWait(wakeTaskStarted);
14643
+ if (emptyLevel === "sibling")
14644
+ yield* harness.offerAndWait(
14645
+ claudeSdkFrame({
14646
+ ...wakeTaskStarted,
14647
+ task_id: secondTaskId,
14648
+ tool_use_id: "toolu_second_owed_wake",
14649
+ uuid: "00000000-0000-4000-8000-000000000205",
14650
+ }),
14651
+ );
14652
+ const awaitBackgroundTasks = harness.runtime.awaitBackgroundTasks;
14653
+ assert.isDefined(awaitBackgroundTasks);
14654
+ yield* awaitBackgroundTasks!({
14655
+ providerThread: harness.providerThread,
14656
+ taskIds,
14657
+ awaited: true,
14658
+ });
14659
+ yield* Queue.offer(harness.sdkMessages, turnOneResult);
14660
+ yield* harness.awaitTerminal(1);
14661
+ const afterFirstTurn = harness.events.length;
14662
+ if (emptyLevel === "before" || emptyLevel === "sibling")
14663
+ yield* harness.offerAndWait(emptyRoster);
14664
+ yield* harness.offerAndWait(wakeNotification);
14665
+ if (emptyLevel === "after") yield* harness.offerAndWait(emptyRoster);
14666
+ // Query replacement reads the internal roster, which the exit already cleared.
14667
+ assert.isFalse(yield* harness.hasPendingBackgroundWork);
14668
+ yield* harness.offerAndWait(wakeTurnInit);
14669
+ yield* harness.runtime.startTurn(
14670
+ makeClaudeTestTurnInput({
14671
+ threadId: harness.threadId,
14672
+ providerThread: harness.providerThread,
14673
+ now,
14674
+ attemptId: wakeAttemptId,
14675
+ text: "Background task completed.",
14676
+ attachments: [],
14677
+ providerTurnOrdinal: 2,
14678
+ messageCreatedBy: "agent",
14679
+ messageCreationSource: "provider",
14680
+ }),
14681
+ );
14682
+ yield* harness.offerAndWait(wakeAssistant);
14683
+ yield* harness.offerAndWait(wakeResult);
14684
+ yield* harness.awaitTerminal(2);
14685
+
14686
+ const wakeStart = harness.events.findIndex(
14687
+ (event) =>
14688
+ event.type === "provider_turn.updated" &&
14689
+ event.providerTurn.runAttemptId === wakeAttemptId,
14690
+ );
14691
+ assert.isAbove(wakeStart, -1);
14692
+ const rosters = harness.events.flatMap((event, index) =>
14693
+ event.type === "provider_thread.updated" &&
14694
+ event.providerThread.id === harness.providerThread.id &&
14695
+ index >= afterFirstTurn
14696
+ ? [
14697
+ {
14698
+ beforeWake: index < wakeStart,
14699
+ pendingWork: event.pendingBackgroundWork === true,
14700
+ taskIds: (event.providerThread.pendingBackgroundTasks ?? []).map((task) => [
14701
+ task.taskId,
14702
+ task.awaited === true,
14703
+ ]),
14704
+ },
14705
+ ]
14706
+ : [],
14707
+ );
14708
+ assert.deepEqual(
14709
+ rosters.filter((roster) => roster.beforeWake).map((roster) => roster.taskIds),
14710
+ rosters
14711
+ .filter((roster) => roster.beforeWake)
14712
+ .map(() => taskIds.map((id) => [id, true])),
14713
+ );
14714
+ assert.isTrue(
14715
+ rosters.filter((roster) => roster.beforeWake).every((roster) => roster.pendingWork),
14716
+ );
14717
+ assert.deepEqual(
14718
+ rosters.at(-1)?.taskIds,
14719
+ emptyLevel === "sibling" ? [[secondTaskId, true]] : [],
14720
+ );
14721
+ if (emptyLevel === "sibling") {
14722
+ assert.isTrue(rosters.at(-1)?.pendingWork);
14723
+ yield* harness.offerAndWait(
14724
+ claudeSdkFrame({
14725
+ ...wakeNotification,
14726
+ task_id: secondTaskId,
14727
+ tool_use_id: "toolu_second_owed_wake",
14728
+ uuid: "00000000-0000-4000-8000-000000000206",
14729
+ }),
14730
+ );
14731
+ yield* harness.offerAndWait(
14732
+ claudeSdkFrame({ ...wakeTurnInit, uuid: "00000000-0000-4000-8000-000000000207" }),
14733
+ );
14734
+ yield* harness.runtime.startTurn(
14735
+ makeClaudeTestTurnInput({
14736
+ threadId: harness.threadId,
14737
+ providerThread: harness.providerThread,
14738
+ now,
14739
+ attemptId: RunAttemptId.make("attempt-claude-owed-sibling-3"),
14740
+ text: "Second command completed.",
14741
+ attachments: [],
14742
+ providerTurnOrdinal: 3,
14743
+ messageCreatedBy: "agent",
14744
+ messageCreationSource: "provider",
14745
+ }),
14746
+ );
14747
+ yield* harness.offerAndWait(
14748
+ makeAssistantTextFrame({
14749
+ uuid: "00000000-0000-4000-8000-000000000208",
14750
+ text: "Second command finished.",
14751
+ }),
14752
+ );
14753
+ yield* harness.offerAndWait(
14754
+ makeResultFrame({
14755
+ uuid: "00000000-0000-4000-8000-000000000209",
14756
+ result: "Second command finished.",
14757
+ origin: { kind: "task-notification" },
14758
+ }),
14759
+ );
14760
+ yield* harness.awaitTerminal(3);
14761
+ assert.deepEqual(
14762
+ providerThreadRosterEvents(harness.events).at(-1)?.providerThread
14763
+ .pendingBackgroundTasks,
14764
+ [],
14765
+ );
14766
+ }
14767
+ }).pipe(
14768
+ Effect.provide(
14769
+ Layer.mergeAll(
14770
+ IdAllocator.layer,
14771
+ McpProviderSessions.layer,
14772
+ NodeServices.layer,
14773
+ NodeCrypto.layer,
14774
+ ),
14775
+ ),
14776
+ ),
14777
+ ),
14778
+ );
14779
+
14780
it.effect.each([true, false])(
14781
"carries a rejected wake rate limit into the continuation failure (event=%s)",
14782
(standaloneEvent) =>
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+71
-12
@@ -6040,6 +6040,24 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6040
taskId,
6041
);
6042
6043
+ // Untraced with roster emission: this runs for each streamed roster update.
6044
+ const owedWakeTasksForNativeThread = Effect.fnUntraced(function* (nativeThreadId: string) {
6045
+ const previous = (yield* Ref.get(lastProviderThreadByNativeThread)).get(nativeThreadId);
6046
+ const roster = rosterForNativeThread(
6047
+ yield* Ref.get(pendingBackgroundTasksByNativeThread),
6048
+ nativeThreadId,
6049
+ );
6050
+ const tasks: OrchestrationV2PendingBackgroundTask[] = [];
6051
+ for (const task of previous?.pendingBackgroundTasks ?? []) {
6052
+ if (
6053
+ !roster.has(task.taskId) &&
6054
+ (yield* isKnownOpaqueBackgroundTaskOnNativeThread(nativeThreadId, task.taskId))
6055
+ )
6056
+ tasks.push(task);
6057
+ }
6058
+ return tasks;
6059
+ });
6060
+
6061
const emitProviderThreadRoster = Effect.fnUntraced(function* (input: {
6062
readonly nativeThreadId: string;
6063
readonly providerThread: OrchestrationV2ProviderThread;
@@ -6053,16 +6071,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6071
yield* Ref.get(awaitedBackgroundTasksByNativeThread),
6072
input.nativeThreadId,
6073
);
6074
+ const owedWakeTasks = yield* owedWakeTasksForNativeThread(input.nativeThreadId);
6075
const now = yield* DateTime.now;
6076
const providerThread: OrchestrationV2ProviderThread = {
6077
...input.providerThread,
6078
providerSessionId: session.id,
6079
...(input.status === undefined ? {} : { status: input.status }),
6061
- pendingBackgroundTasks: publishedPendingBackgroundTasks(
6062
- input.nativeThreadId,
6063
- roster,
6064
- awaitedTaskIds,
6065
- ),
6080
+ pendingBackgroundTasks: [
6081
+ ...publishedPendingBackgroundTasks(input.nativeThreadId, roster, awaitedTaskIds),
6082
+ ...owedWakeTasks,
6083
+ ],
6084
...(goalsByNativeThread.has(input.nativeThreadId)
6085
? { goal: goalsByNativeThread.get(input.nativeThreadId) ?? null }
6086
: {}),
@@ -6073,6 +6091,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
6091
type: "provider_thread.updated",
6092
driver: CLAUDE_PROVIDER,
6093
providerThread,
6094
+ pendingBackgroundWork: owedWakeTasks.length > 0,
6095
});
6096
});
6097
@@ -8665,6 +8684,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8684
yield* Ref.get(awaitedBackgroundTasksByNativeThread),
8685
nativeThreadId,
8686
);
8687
+ const owedWakeTasks =
8688
+ nativeThreadId === null
8689
+ ? []
8690
+ : yield* owedWakeTasksForNativeThread(nativeThreadId);
8691
// A stopped turn that wrote to the transcript moved past the pinned head.
8692
const clearConversationHead =
8693
(input.status === "completed" || input.context.nativeMessageCursor !== null) &&
@@ -8703,11 +8726,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8726
input.context.input.providerThread.firstRunOrdinal ??
8727
input.context.input.runOrdinal,
8728
lastRunOrdinal: input.context.input.runOrdinal,
8706
- pendingBackgroundTasks: publishedPendingBackgroundTasks(
8707
- nativeThreadId,
8708
- roster,
8709
- awaitedTaskIds,
8710
- ),
8729
+ pendingBackgroundTasks: [
8730
+ ...publishedPendingBackgroundTasks(nativeThreadId, roster, awaitedTaskIds),
8731
+ ...owedWakeTasks,
8732
+ ],
8733
status: input.status === "completed" ? "active" : "idle",
8734
updatedAt: input.completedAt,
8735
};
@@ -8716,6 +8738,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8738
type: "provider_thread.updated" as const,
8739
driver: CLAUDE_PROVIDER,
8740
providerThread,
8741
+ pendingBackgroundWork: owedWakeTasks.length > 0,
8742
});
8743
}),
8744
emitProviderEvent(terminalEvent),
@@ -9235,6 +9258,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9258
...later,
9259
];
9260
if (!(yield* replaceWakeBuffer(nativeThreadId, buffered, kept, retiredTaskIds))) return;
9261
+ for (const taskId of retiredTaskIds) {
9262
+ yield* clearPendingBackgroundTask(nativeThreadId, taskId);
9263
+ yield* clearTaskIdFromNativeThreadSet(
9264
+ wakeEligibleBackgroundTasksByNativeThread,
9265
+ nativeThreadId,
9266
+ taskId,
9267
+ );
9268
+ yield* clearOpaqueBackgroundTaskReplayTombstone(nativeThreadId, taskId);
9269
+ }
9270
+ yield* endClaudeMonitorTasks((taskId) => retiredTaskIds.has(taskId));
9271
+ const providerThread = (yield* Ref.get(lastProviderThreadByNativeThread)).get(
9272
+ nativeThreadId,
9273
+ );
9274
+ if (providerThread !== undefined)
9275
+ yield* emitProviderThreadRoster({ nativeThreadId, providerThread });
9276
if (latch.end === undefined) {
9277
// A lifecycle result closes the root frames before it without opening the wake's turn.
9278
const sinceLifecycleResult = retired.slice(
@@ -9355,7 +9393,11 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9393
: baseThread.status === "idle"
9394
? ("active" as const)
9395
: baseThread.status;
9358
- yield* emitProviderThreadRoster({ nativeThreadId, providerThread: baseThread, status });
9396
+ yield* emitProviderThreadRoster({
9397
+ nativeThreadId,
9398
+ providerThread: baseThread,
9399
+ status,
9400
+ });
9401
});
9402
9403
const applyBackgroundTaskRosterMessage = Effect.fnUntraced(function* (input: {
@@ -11961,7 +12003,24 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
12003
});
12004
return updated;
12005
});
11964
- yield* rememberProviderThread(turnInput.providerThread);
12006
+ // A queued turn's snapshot can predate another task's exit.
12007
+ const owedWakeTasks = yield* owedWakeTasksForNativeThread(nativeThreadId);
12008
+ yield* rememberProviderThread(
12009
+ owedWakeTasks.length === 0
12010
+ ? turnInput.providerThread
12011
+ : {
12012
+ ...turnInput.providerThread,
12013
+ pendingBackgroundTasks: [
12014
+ ...(turnInput.providerThread.pendingBackgroundTasks ?? []),
12015
+ ...owedWakeTasks.filter(
12016
+ (task) =>
12017
+ !(turnInput.providerThread.pendingBackgroundTasks ?? []).some(
12018
+ (current) => current.taskId === task.taskId,
12019
+ ),
12020
+ ),
12021
+ ],
12022
+ },
12023
+ );
12024
if (!goalsByNativeThread.has(nativeThreadId)) {
12025
goalsByNativeThread.set(nativeThreadId, turnInput.providerThread.goal ?? null);
12026
}
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+314
-102
@@ -1,3 +1,6 @@
1
+import * as Exit from "effect/Exit";
2
+import * as EffectOutbox from "./EffectOutbox.ts";
3
+import * as EffectWorker from "./EffectWorker.ts";
4
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
5
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
6
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -118,6 +121,7 @@ const layerTest = Layer.mergeAll(
121
RuntimeLayer.layer,
122
RuntimeLayer.layerEventSink,
123
ThreadCommandExecutor.layer,
124
+ EffectOutbox.layer,
125
).pipe(
126
Layer.provideMerge(RuntimeLayer.layerProjectService),
127
Layer.provide(
@@ -2428,138 +2432,344 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
2432
}),
2433
);
2434
2431
- it.effect("delivers the run a background command's wake starts after the original result", () =>
2435
+ it.effect("checkpoint retry releases a pending sibling after its wake offer failed", () =>
2436
Effect.gen(function* () {
2437
const orchestrator = yield* Orchestrator.OrchestratorV2;
2434
- const eventSink = yield* EventSink.EventSinkV2;
2438
+ const sink = yield* EventSink.EventSinkV2;
2439
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
2440
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2441
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2442
const now = yield* DateTime.now;
2437
- const parentThreadId = ThreadId.make("thread:background-wake-parent");
2438
- const projectId = ProjectId.make("project:background-wake");
2439
- const parentRunId = RunId.make("run:background-wake-parent");
2440
- const rootNodeId = NodeId.make("node:background-wake-parent");
2443
+ const threadId = ThreadId.make("thread:checkpoint-own-delivery");
2444
+ const originRunId = RunId.make("run:checkpoint-own-delivery:origin");
2445
+ const firstId = NodeId.make("node:checkpoint-own-delivery:first");
2446
+ const siblingId = NodeId.make("node:checkpoint-own-delivery:sibling");
2447
yield* seedParentWithTerminalTask({
2442
- threadId: parentThreadId,
2443
- projectId,
2444
- runId: parentRunId,
2445
- rootNodeId,
2446
- taskId: NodeId.make("node:background-wake-seed"),
2447
- deliveryState: "acknowledged",
2448
- now,
2449
- });
2450
- const child = yield* seedRestartCancelledChild({
2451
- parentThreadId,
2452
- projectId,
2453
- parentRunId,
2454
- rootNodeId,
2455
- name: "background-wake-child",
2456
- completionWake: "always",
2457
- continuationPending: false,
2458
- runStatus: "completed",
2448
+ threadId,
2449
+ projectId: ProjectId.make("project:checkpoint-own-delivery"),
2450
+ runId: originRunId,
2451
+ rootNodeId: NodeId.make("node:checkpoint-own-delivery:root"),
2452
+ taskId: firstId,
2453
+ deliveryState: "claimed",
2454
+ deliveryTaskIds: [firstId],
2455
+ parentRunStatus: "completed",
2456
now,
2457
});
2461
- yield* orchestrator.recoverDelegatedTasks;
2462
- let firstWake = yield* requests.take;
2463
- while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
2464
- yield* orchestrator.dispatch({
2465
- type: "delegated_task.completion-delivery.acknowledge",
2466
- commandId: CommandId.make("command:background-wake-ack"),
2467
- parentThreadId,
2468
- taskId: child.taskId,
2469
- observedByRunId: parentRunId,
2470
- });
2471
- const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2472
- (run) => run.id === parentRunId,
2473
- )!;
2474
- yield* eventSink.write({
2475
- commandId: reconcileCommandId("background-wake-parent-finished"),
2476
- events: [
2477
- {
2478
- id: EventId.make("event:background-wake-parent-finished"),
2479
- type: "run.updated",
2480
- threadId: parentThreadId,
2481
- runId: parentRunId,
2482
- occurredAt: now,
2483
- payload: { ...parentRun, status: "completed", completedAt: now },
2484
- },
2485
- ],
2486
- });
2487
-
2488
- // The shape ProviderContinuationService dispatches when the provider
2489
- // reports a background command (wire kind `background_command`) ended.
2490
- const wakeMessageId = MessageId.make("message:background-wake");
2458
+ const original = yield* orchestrator.getThreadProjection(threadId);
2459
+ const first = original.subagents.find((task) => task.id === firstId)!;
2460
+ const delivery = original.runs.find((run) => run.id === originRunId)!.delegatedCompletion!
2461
+ .delivery!;
2462
yield* orchestrator.dispatch({
2463
type: "message.dispatch",
2464
createdBy: "agent",
2494
- creationSource: "provider",
2495
- commandId: CommandId.make("command:background-wake"),
2496
- threadId: child.childThreadId,
2497
- messageId: wakeMessageId,
2498
- text: "Background activity updated",
2499
- notification: {
2500
- source: { kind: "command" },
2501
- outcome: "completed",
2502
- summary: 'Command "sleep 60" finished',
2503
- },
2465
+ creationSource: "server",
2466
+ commandId: reconcileCommandId("checkpoint-own-delivery-wake"),
2467
+ threadId,
2468
+ messageId: delivery.messageId,
2469
+ text: "First result",
2470
attachments: [],
2471
dispatchMode: { type: "queue_after_active" },
2472
+ delegatedCompletion: {
2473
+ parentRunId: originRunId,
2474
+ generation: delivery.generation,
2475
+ taskIds: delivery.taskIds,
2476
+ },
2477
});
2507
- const run = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
2508
- (row) => row.userMessageId === wakeMessageId,
2478
+ const started = yield* orchestrator.getThreadProjection(threadId);
2479
+ const run = started.runs.find(
2480
+ (candidate) => candidate.userMessageId === delivery.messageId,
2481
)!;
2510
- assert.isDefined(run.delegatedTaskId);
2511
- yield* eventSink.write({
2512
- commandId: reconcileCommandId("background-wake-finished"),
2482
+ const scopeId = started.nodes.find(
2483
+ (node) => node.id === run.rootNodeId,
2484
+ )!.checkpointScopeId!;
2485
+ yield* outbox.cancelUnsettled({
2486
+ threadId,
2487
+ effectTypes: ["provider-turn.start"],
2488
+ reason: "The delivery turn already completed.",
2489
+ });
2490
+ const commandId = reconcileCommandId("checkpoint-own-delivery-finished");
2491
+ yield* sink.writeWithEffects({
2492
+ commandId,
2493
events: [
2494
{
2515
- id: EventId.make("event:background-wake-result"),
2516
- type: "message.updated",
2517
- threadId: child.childThreadId,
2518
- runId: run.id,
2495
+ id: EventId.make("event:checkpoint-own-delivery:sibling"),
2496
+ type: "subagent.updated",
2497
+ threadId,
2498
+ runId: originRunId,
2499
+ nodeId: siblingId,
2500
occurredAt: now,
2501
payload: {
2521
- id: MessageId.make("message:background-wake-result"),
2522
- threadId: child.childThreadId,
2523
- runId: run.id,
2524
- nodeId: null,
2525
- role: "assistant",
2526
- text: "The background command finished cleanly.",
2527
- attachments: [],
2528
- streaming: false,
2529
- createdBy: "agent",
2530
- creationSource: "server",
2531
- createdAt: now,
2532
- updatedAt: now,
2502
+ ...first,
2503
+ id: siblingId,
2504
+ completionDelivery: { state: "pending", observedByRunId: null },
2505
},
2506
},
2507
{
2536
- id: EventId.make("event:background-wake-finished"),
2508
+ id: EventId.make("event:checkpoint-own-delivery:finished"),
2509
type: "run.updated",
2538
- threadId: child.childThreadId,
2510
+ threadId,
2511
runId: run.id,
2512
occurredAt: now,
2541
- payload: { ...run, status: "completed", startedAt: now, completedAt: now },
2513
+ payload: {
2514
+ ...run,
2515
+ status: "completed",
2516
+ startedAt: now,
2517
+ completedAt: now,
2518
+ checkpointId: CheckpointId.make("checkpoint:own-delivery"),
2519
+ },
2520
+ },
2521
+ ],
2522
+ effects: [
2523
+ {
2524
+ id: "effect:checkpoint-own-delivery",
2525
+ commandId,
2526
+ threadId,
2527
+ request: { type: "checkpoint.capture", runId: run.id, scopeId },
2528
},
2529
],
2530
});
2545
- yield* orchestrator.recoverDelegatedTasks;
2546
- const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2547
- (task) => task.id === run.delegatedTaskId,
2548
- )!;
2549
- assert.equal(followup.status, "completed");
2550
- assert.equal(followup.result, "The background command finished cleanly.");
2551
- assert.equal(followup.completionDelivery?.state, "claimed");
2531
+ using offer = vi
2532
+ .spyOn(requests, "offer")
2533
+ .mockImplementationOnce(() => Effect.die("successor wake offer failed"));
2534
+ const firstAttempt = yield* Effect.exit(
2535
+ orchestrator.recoverDelegatedTask(threadId, { completedRunId: run.id }),
2536
+ );
2537
+ assert.isTrue(Exit.isFailure(firstAttempt));
2538
+ const reserved = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
2539
+ (candidate) => candidate.id === originRunId,
2540
+ )!.delegatedCompletion!.delivery!;
2541
+ assert.deepEqual(reserved.taskIds, [siblingId]);
2542
+ yield* worker.drain();
2543
+ assert.lengthOf(offer.mock.calls, 2);
2544
+ const completed = yield* orchestrator.getThreadProjection(threadId);
2545
+ assert.equal(
2546
+ completed.subagents.find((task) => task.id === firstId)?.completionDelivery?.state,
2547
+ "delivered",
2548
+ );
2549
+ assert.equal(
2550
+ completed.subagents.find((task) => task.id === siblingId)?.completionDelivery?.state,
2551
+ "claimed",
2552
+ );
2553
+ assert.deepEqual(
2554
+ completed.runs.find((candidate) => candidate.id === originRunId)?.delegatedCompletion
2555
+ ?.delivery?.taskIds,
2556
+ [siblingId],
2557
+ );
2558
let wake = yield* requests.take;
2553
- while (
2554
- wake.threadId !== parentThreadId ||
2555
- wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId
2556
- ) {
2557
- wake = yield* requests.take;
2558
- }
2559
- assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId);
2559
+ while (wake.threadId !== threadId) wake = yield* requests.take;
2560
+ assert.equal(wake.delegatedCompletion?.generation, delivery.generation + 1);
2561
}),
2562
);
2563
2564
+ it.effect.each(["live", "checkpoint-retry", "checkpoint-offer-retry"] as const)(
2565
+ "delivers the run a background command's wake starts after the original result (%s)",
2566
+ (delivery) =>
2567
+ Effect.gen(function* () {
2568
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
2569
+ const eventSink = yield* EventSink.EventSinkV2;
2570
+ const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
2571
+ const now = yield* DateTime.now;
2572
+ const parentThreadId = ThreadId.make(`thread:background-wake-parent-${delivery}`);
2573
+ const projectId = ProjectId.make(`project:background-wake-${delivery}`);
2574
+ const parentRunId = RunId.make(`run:background-wake-parent-${delivery}`);
2575
+ const rootNodeId = NodeId.make(`node:background-wake-parent-${delivery}`);
2576
+ yield* seedParentWithTerminalTask({
2577
+ threadId: parentThreadId,
2578
+ projectId,
2579
+ runId: parentRunId,
2580
+ rootNodeId,
2581
+ taskId: NodeId.make(`node:background-wake-seed-${delivery}`),
2582
+ deliveryState: "acknowledged",
2583
+ now,
2584
+ });
2585
+ const child = yield* seedRestartCancelledChild({
2586
+ parentThreadId,
2587
+ projectId,
2588
+ parentRunId,
2589
+ rootNodeId,
2590
+ name: `background-wake-child-${delivery}`,
2591
+ completionWake: "always",
2592
+ continuationPending: false,
2593
+ runStatus: "completed",
2594
+ now,
2595
+ });
2596
+ yield* orchestrator.recoverDelegatedTasks;
2597
+ let firstWake = yield* requests.take;
2598
+ while (firstWake.threadId !== parentThreadId) firstWake = yield* requests.take;
2599
+ yield* orchestrator.dispatch({
2600
+ type: "delegated_task.completion-delivery.acknowledge",
2601
+ commandId: CommandId.make(`command:background-wake-ack-${delivery}`),
2602
+ parentThreadId,
2603
+ taskId: child.taskId,
2604
+ observedByRunId: parentRunId,
2605
+ });
2606
+ const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2607
+ (run) => run.id === parentRunId,
2608
+ )!;
2609
+ yield* eventSink.write({
2610
+ commandId: reconcileCommandId(`background-wake-parent-finished-${delivery}`),
2611
+ events: [
2612
+ {
2613
+ id: EventId.make(`event:background-wake-parent-finished-${delivery}`),
2614
+ type: "run.updated",
2615
+ threadId: parentThreadId,
2616
+ runId: parentRunId,
2617
+ occurredAt: now,
2618
+ payload: { ...parentRun, status: "completed", completedAt: now },
2619
+ },
2620
+ ],
2621
+ });
2622
+
2623
+ // The shape ProviderContinuationService dispatches when the provider
2624
+ // reports a background command (wire kind `background_command`) ended.
2625
+ const wakeMessageId = MessageId.make(`message:background-wake-${delivery}`);
2626
+ yield* orchestrator.dispatch({
2627
+ type: "message.dispatch",
2628
+ createdBy: "agent",
2629
+ creationSource: "provider",
2630
+ commandId:
2631
+ delivery === "live"
2632
+ ? CommandId.make(`command:background-wake-${delivery}`)
2633
+ : reconcileCommandId(`background-wake-${delivery}`),
2634
+ threadId: child.childThreadId,
2635
+ messageId: wakeMessageId,
2636
+ text: "Background activity updated",
2637
+ notification: {
2638
+ source: { kind: "command" },
2639
+ outcome: "completed",
2640
+ summary: 'Command "sleep 60" finished',
2641
+ },
2642
+ attachments: [],
2643
+ dispatchMode: { type: "queue_after_active" },
2644
+ });
2645
+ const childProjection = yield* orchestrator.getThreadProjection(child.childThreadId);
2646
+ const run = childProjection.runs.find((row) => row.userMessageId === wakeMessageId)!;
2647
+ const scopeId = childProjection.nodes.find(
2648
+ (node) => node.id === run.rootNodeId,
2649
+ )!.checkpointScopeId!;
2650
+ const commandId =
2651
+ delivery === "live"
2652
+ ? CommandId.make(`command:background-wake-finished-${delivery}`)
2653
+ : reconcileCommandId(`background-wake-finished-${delivery}`);
2654
+ if (delivery !== "live") {
2655
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
2656
+ yield* outbox.cancelUnsettled({
2657
+ threadId: child.childThreadId,
2658
+ effectTypes: ["provider-turn.start"],
2659
+ reason: "The turn already finished before its checkpoint retry.",
2660
+ });
2661
+ }
2662
+ const afterSequence = yield* eventSink.latestSequence();
2663
+ assert.isDefined(run.delegatedTaskId);
2664
+ yield* eventSink.writeWithEffects({
2665
+ commandId,
2666
+ effects:
2667
+ delivery === "live"
2668
+ ? []
2669
+ : [
2670
+ {
2671
+ id: `effect:background-wake-checkpoint-retry-${delivery}`,
2672
+ commandId,
2673
+ threadId: child.childThreadId,
2674
+ request: { type: "checkpoint.capture", runId: run.id, scopeId },
2675
+ },
2676
+ ],
2677
+ events: [
2678
+ {
2679
+ id: EventId.make(`event:background-wake-result-${delivery}`),
2680
+ type: "message.updated",
2681
+ threadId: child.childThreadId,
2682
+ runId: run.id,
2683
+ occurredAt: now,
2684
+ payload: {
2685
+ id: MessageId.make(`message:background-wake-result-${delivery}`),
2686
+ threadId: child.childThreadId,
2687
+ runId: run.id,
2688
+ nodeId: null,
2689
+ role: "assistant",
2690
+ text: "The background command finished cleanly.",
2691
+ attachments: [],
2692
+ streaming: false,
2693
+ createdBy: "agent",
2694
+ creationSource: "server",
2695
+ createdAt: now,
2696
+ updatedAt: now,
2697
+ },
2698
+ },
2699
+ {
2700
+ id: EventId.make(`event:background-wake-finished-${delivery}`),
2701
+ type: "run.updated",
2702
+ threadId: child.childThreadId,
2703
+ runId: run.id,
2704
+ occurredAt: now,
2705
+ payload: {
2706
+ ...run,
2707
+ status: "completed",
2708
+ startedAt: now,
2709
+ completedAt: now,
2710
+ checkpointId:
2711
+ delivery === "live"
2712
+ ? null
2713
+ : CheckpointId.make(`checkpoint:background-wake-${delivery}`),
2714
+ },
2715
+ },
2716
+ ],
2717
+ });
2718
+ if (delivery !== "live") {
2719
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2720
+ if (delivery === "checkpoint-offer-retry") {
2721
+ using offer = vi
2722
+ .spyOn(requests, "offer")
2723
+ .mockImplementationOnce(() => Effect.die("wake offer failed"));
2724
+ const first = yield* Effect.exit(
2725
+ orchestrator.recoverDelegatedTask(child.childThreadId),
2726
+ );
2727
+ assert.isTrue(Exit.isFailure(first));
2728
+ const committed = (yield* orchestrator.getThreadProjection(
2729
+ parentThreadId,
2730
+ )).subagents.find((task) => task.id === run.delegatedTaskId)!;
2731
+ assert.equal(committed.result, "The background command finished cleanly.");
2732
+ yield* worker.drain();
2733
+ assert.lengthOf(offer.mock.calls, 2);
2734
+ } else {
2735
+ yield* worker.drain();
2736
+ }
2737
+ } else {
2738
+ yield* eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
2739
+ Stream.filter(
2740
+ (stored) =>
2741
+ stored.event.type === "subagent.updated" &&
2742
+ stored.event.payload.id === run.delegatedTaskId &&
2743
+ stored.event.payload.status === "completed",
2744
+ ),
2745
+ Stream.take(1),
2746
+ Stream.runDrain,
2747
+ );
2748
+ }
2749
+ const completed = yield* orchestrator.getThreadProjection(parentThreadId);
2750
+ const followup = completed.subagents.find((task) => task.id === run.delegatedTaskId)!;
2751
+ assert.lengthOf(
2752
+ completed.contextTransfers.filter(
2753
+ (transfer) =>
2754
+ transfer.type === "subagent_result" &&
2755
+ transfer.sourceThreadId === child.childThreadId,
2756
+ ),
2757
+ 2,
2758
+ );
2759
+ assert.equal(followup.status, "completed");
2760
+ assert.equal(followup.result, "The background command finished cleanly.");
2761
+ assert.equal(followup.completionDelivery?.state, "claimed");
2762
+ let wake = yield* requests.take;
2763
+ while (
2764
+ wake.threadId !== parentThreadId ||
2765
+ wake.delegatedCompletion?.messageId === firstWake.delegatedCompletion?.messageId
2766
+ ) {
2767
+ wake = yield* requests.take;
2768
+ }
2769
+ assert.equal(wake.delegatedCompletion?.parentRunId, parentRunId);
2770
+ }),
2771
+ );
2772
+
2773
// The child's follow-up run sends `t3_thread_send` to its own thread before it ends.
2774
const startSelfQueuedFollowup = (name: string) =>
2775
Effect.gen(function* () {
@@ -3671,7 +3881,9 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
3881
assert.isTrue(yield* orchestrator.delegatedTaskResultPending(recut.childThreadId));
3882
assert.isTrue(yield* orchestrator.delegatedTaskResultPending(resumed.childThreadId));
3883
assert.isFalse(yield* orchestrator.delegatedTaskResultPending(stopped.childThreadId));
3674
- yield* orchestrator.recoverDelegatedTask(recut.childThreadId, recut.childRunId);
3884
+ yield* orchestrator.recoverDelegatedTask(recut.childThreadId, {
3885
+ sourceRunId: recut.childRunId,
3886
+ });
3887
const replayed = yield* orchestrator.getThreadProjection(threadId);
3888
assert.equal(replayed.subagents.find((row) => row.id === recut.taskId)?.status, "running");
3889
yield* eventSink.write({
apps/server/src/orchestration-v2/EffectWorker.ts
+7
-1
@@ -106,7 +106,7 @@ export const layerExecutor: Layer.Layer<
106
Effect.tapError(() =>
107
willRetry
108
? Effect.void
109
- : threads.recoverDelegatedTask(effect.threadId, sourceRunId),
109
+ : threads.recoverDelegatedTask(effect.threadId, { sourceRunId }),
110
),
111
Effect.mapError(
112
(cause) =>
@@ -506,6 +506,12 @@ export const layerExecutor: Layer.Layer<
506
scopeId: effect.request.scopeId,
507
})
508
.pipe(
509
+ // Completion bookkeeping must survive a missed live run update, including capture retries.
510
+ Effect.andThen(
511
+ threads.recoverDelegatedTask(effect.threadId, {
512
+ completedRunId: effect.request.runId,
513
+ }),
514
+ ),
515
Effect.mapError(
516
(cause) =>
517
new OrchestrationEffectExecutionError({
apps/server/src/orchestration-v2/Orchestrator.ts
+40
-12
@@ -382,7 +382,10 @@ export interface OrchestratorV2DispatchResult {
382
export interface OrchestratorV2Shape {
383
readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
384
readonly recoverDelegatedTasks: Effect.Effect<void>;
385
- readonly recoverDelegatedTask: (threadId: ThreadId, sourceRunId: RunId) => Effect.Effect<void>;
385
+ readonly recoverDelegatedTask: (
386
+ threadId: ThreadId,
387
+ options?: { readonly sourceRunId?: RunId; readonly completedRunId?: RunId },
388
+ ) => Effect.Effect<void, OrchestratorProjectionError>;
389
/** Commits cancellation with the child's Stop after descendants accept stopping. */
390
readonly cancelDelegatedTask: <E, R>(input: {
391
readonly parentThreadId: ThreadId;
@@ -15054,23 +15057,48 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15057
);
15058
15059
const recoverDelegatedTask = Effect.fn("Orchestrator.recoverDelegatedTask")(
15057
- function* (threadId: ThreadId, sourceRunId: RunId) {
15060
+ function* (
15061
+ threadId: ThreadId,
15062
+ options?: { readonly sourceRunId?: RunId; readonly completedRunId?: RunId },
15063
+ ) {
15064
+ const completedRunId = options?.completedRunId;
15065
+ if (completedRunId !== undefined) {
15066
+ yield* threadDispatch.withLock(
15067
+ threadId,
15068
+ Effect.gen(function* () {
15069
+ const message = yield* projectionStore.getRunMessage(threadId, completedRunId);
15070
+ if (message?.delegatedCompletion === undefined) return;
15071
+ // A prior attempt may have committed the next generation before offering its wake.
15072
+ yield* offerDelegatedCompletionDelivery(
15073
+ threadId,
15074
+ message.delegatedCompletion.parentRunId,
15075
+ );
15076
+ yield* finalizeDelegatedCompletionDelivery(threadId, completedRunId);
15077
+ }),
15078
+ );
15079
+ }
15080
+ const sourceRunId = options?.sourceRunId;
15081
const parentThreadId = yield* appOwnedSubagentParentThreadId(threadId);
15082
if (parentThreadId === undefined) return;
15083
+ if (
15084
+ sourceRunId === undefined &&
15085
+ !(yield* projectionStore.hasPendingDelegatedCompletion(threadId))
15086
+ )
15087
+ return;
15088
yield* threadDispatch.withLock(
15089
parentThreadId,
15062
- finalizeAppOwnedSubagent(threadId, { settledContinuationOf: sourceRunId }),
15090
+ offerDelegatedCompletionDeliveries(parentThreadId).pipe(
15091
+ Effect.andThen(
15092
+ finalizeAppOwnedSubagent(
15093
+ threadId,
15094
+ sourceRunId === undefined ? undefined : { settledContinuationOf: sourceRunId },
15095
+ ),
15096
+ ),
15097
+ ),
15098
);
15099
},
15065
- (effect, threadId, _sourceRunId) =>
15066
- effect.pipe(
15067
- Effect.catchCause((cause) =>
15068
- Effect.logWarning("Failed to recover delegated task after restart", {
15069
- childThreadId: threadId,
15070
- cause,
15071
- }),
15072
- ),
15073
- ),
15100
+ (effect, threadId, _options) =>
15101
+ effect.pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
15102
);
15103
15104
const cancelDelegatedTask: OrchestratorV2Shape["cancelDelegatedTask"] = Effect.fn(
apps/server/src/orchestration-v2/ProjectionStore.ts
+18
@@ -477,6 +477,9 @@ export interface ProjectionStoreV2Shape {
477
readonly listAppOwnedChildThreadIds: (
478
threadId: ThreadId,
479
) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
480
+ readonly hasPendingDelegatedCompletion: (
481
+ childThreadId: ThreadId,
482
+ ) => Effect.Effect<boolean, ProjectionStoreV2Error>;
483
readonly getNextTurnItemOrdinal: (
484
threadId: ThreadId,
485
) => Effect.Effect<number, ProjectionStoreV2Error>;
@@ -5809,6 +5812,20 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5812
Effect.map((rows) => [...new Set(rows.map((row) => ThreadId.make(row.child_thread_id)))]),
5813
Effect.mapError(controlReadError(threadId)),
5814
);
5815
+ const hasPendingDelegatedCompletion: ProjectionStoreV2Shape["hasPendingDelegatedCompletion"] =
5816
+ Effect.fn("ProjectionStore.hasPendingDelegatedCompletion")((childThreadId) =>
5817
+ sql`SELECT 1 FROM orchestration_v2_projection_subagents AS task
5818
+ LEFT JOIN orchestration_v2_projection_runs AS owner ON owner.run_id = task.run_id
5819
+ WHERE task.child_thread_id = ${childThreadId} AND task.origin = 'app_owned'
5820
+ AND (json_extract(task.payload_json, '$.result') IS NULL
5821
+ OR (task.completion_delivery_state = 'claimed'
5822
+ AND json_extract(owner.payload_json, '$.delegatedCompletion.disposition') = 'open'
5823
+ AND json_type(owner.payload_json, '$.delegatedCompletion.delivery') = 'object'))
5824
+ LIMIT 1`.pipe(
5825
+ Effect.map((rows) => rows.length > 0),
5826
+ Effect.mapError(controlReadError(childThreadId)),
5827
+ ),
5828
+ );
5829
const getNextTurnItemOrdinal: ProjectionStoreV2Shape["getNextTurnItemOrdinal"] = (threadId) =>
5830
sql<{ ordinal: number | null }>`SELECT MAX(ordinal) AS ordinal
5831
FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId}`.pipe(
@@ -7832,6 +7849,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7849
getMessageCount: readOnReader(getMessageCount),
7850
getSubagentCount: readOnReader(getSubagentCount),
7851
listAppOwnedChildThreadIds: readOnReader(listAppOwnedChildThreadIds),
7852
+ hasPendingDelegatedCompletion: readOnReader(hasPendingDelegatedCompletion),
7853
getNextTurnItemOrdinal: readOnReader(getNextTurnItemOrdinal),
7854
getTurnItem: readOnReader(getTurnItem),
7855
getMessagesById: readOnReader(getMessagesById),
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+1
@@ -239,6 +239,7 @@ it.effect(
239
getThreadHistoryPage: () => Effect.die("unused"),
240
getTimelinePage: () => Effect.die("Unused timeline read"),
241
getMessageCount: () => Effect.die("unused message count"),
242
+ hasPendingDelegatedCompletion: () => Effect.die("unused hasPendingDelegatedCompletion"),
243
getSubagentCount: () => Effect.die("unused subagent count"),
244
listAppOwnedChildThreadIds: () => Effect.die("unused listAppOwnedChildThreadIds"),
245
getNextTurnItemOrdinal: () => Effect.die("unused ordinal read"),
apps/server/src/orchestration-v2/RestartContinuation.ts
+1
-1
@@ -352,7 +352,7 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
352
Effect.andThen(
353
Effect.gen(function* () {
354
const threads = yield* ThreadManagementService.ThreadManagementService;
355
- yield* threads.recoverDelegatedTask(input.threadId, input.sourceRunId);
355
+ yield* threads.recoverDelegatedTask(input.threadId, { sourceRunId: input.sourceRunId });
356
}),
357
),
358
),
apps/server/src/orchestration-v2/RunFinalizationQueue.test.ts
+3
-1
@@ -94,7 +94,9 @@ it.effect.each([
94
Layer.mock(CheckpointRollback.CheckpointRollbackServiceV2)({}),
95
Layer.mock(RuntimeRequests.RuntimeRequestServiceV2)({}),
96
Layer.mock(ThreadTitles.ThreadTitleRegenerationService)({}),
97
- Layer.mock(ThreadManagement.ThreadManagementService)({}),
97
+ Layer.mock(ThreadManagement.ThreadManagementService)({
98
+ recoverDelegatedTask: () => Effect.void,
99
+ }),
100
ServerSettings.layerTest(),
101
Layer.mock(ProviderTurnStart.ProviderTurnStartServiceV2)({
102
start: () => Effect.sync(() => calls.push("start")).pipe(Effect.asVoid),
apps/server/src/orchestration-v2/ThreadStop.test.ts
+2
-2
@@ -1242,7 +1242,7 @@ it.effect.each([
1242
(projection) => projection.runtimeRequests[0],
1243
);
1244
1245
- yield* orchestrator.recoverDelegatedTask(childThreadId, childRun.id);
1245
+ yield* orchestrator.recoverDelegatedTask(childThreadId, { sourceRunId: childRun.id });
1246
assert.equal((yield* request)?.status, "pending");
1247
if (ending === "interrupted") {
1248
yield* orchestrator.dispatch({
@@ -1260,7 +1260,7 @@ it.effect.each([
1260
payload: { ...childRun, status: ending, startedAt: now, completedAt: now },
1261
});
1262
}
1263
- yield* orchestrator.recoverDelegatedTask(childThreadId, childRun.id);
1263
+ yield* orchestrator.recoverDelegatedTask(childThreadId, { sourceRunId: childRun.id });
1264
assert.equal((yield* request)?.status, question);
1265
assert.equal((yield* request)?.decision, question === "resolved" ? "cancel" : undefined);
1266
}).pipe(Effect.provide(layerTest)),
apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/claude_transcript.ndjson
+3
-1
@@ -1,4 +1,4 @@
1
-{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"claude_background_task_after_root","metadata":{"prompts":["Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."],"model":"claude-sonnet-4-6","nativeSessionId":"ec24a006-617b-4324-b516-57a7afff15a4","queryMode":"streaming","tools":"claude_code","permissionMode":"bypassPermissions","timingCompression":"25-second background gap controlled by the replay scenario gate","generatedBy":"manual-replay-from-thread-ec24a006 (ctm-catchup-l2-20260729, captured 2026-07-30T00:35:14Z)"}}
1
+{"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"claude_background_task_after_root","metadata":{"prompts":["Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."],"model":"claude-sonnet-4-6","nativeSessionId":"ec24a006-617b-4324-b516-57a7afff15a4","queryMode":"streaming","tools":"claude_code","permissionMode":"bypassPermissions","timingCompression":"25-second background gap controlled by the replay scenario gate","generatedBy":"manual-replay-from-thread-ec24a006 (ctm-catchup-l2-20260729, captured 2026-07-30T00:35:14Z)","syntheticExtension":"Wake assistant and result frames extend the captured notification-only transcript to exercise completion consumption."}}
2
{"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"bypassPermissions","allowDangerouslySkipPermissions":true,"sessionId":"ec24a006-617b-4324-b516-57a7afff15a4","settings":{"showThinkingSummaries":true}}}}
3
{"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Live-test post-settle background Bash wake. Do exactly this in order, with no extra steps.\n\n1) Run this exact command using the Bash tool with run_in_background set to true:\n sleep 25 && echo L2_BG_DONE\n2) Immediately after starting it, reply with a short message containing exactly L2_STARTED and stop.\n3) Do NOT poll TaskOutput. Do NOT wait for the task. Do not spawn subagents or monitors.\n4) If its completion is reported later, acknowledge it once by replying with exactly L2_WAKE: L2_BG_DONE and stop.\n\nThe point is that your first turn ends while the command is still running."},"parent_tool_use_id":null}}}
4
{"type":"emit_inbound","label":"system:init","frame":{"type":"system","subtype":"init","agents":[],"apiKeySource":"none","claude_code_version":"2.1.219","cwd":"/tmp/claude-replay-claude_background_task_after_root","tools":[],"mcp_servers":[],"model":"claude-sonnet-4-6","permissionMode":"bypassPermissions","slash_commands":[],"output_style":"default","skills":[],"plugins":[],"fast_mode_state":"off","uuid":"fdd8c2b3-f443-405c-996d-1a5ac798e100","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}}
@@ -10,4 +10,6 @@
10
{"type":"emit_inbound","label":"result:root","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":7125,"duration_api_ms":6500,"num_turns":2,"result":"L2_STARTED","stop_reason":"end_turn","session_id":"ec24a006-617b-4324-b516-57a7afff15a4","total_cost_usd":0.001,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":2,"output_tokens":9,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":2,"outputTokens":9,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"633b885b-240f-4f44-bfe1-d554088b2dfd"}}
11
{"type":"emit_inbound","label":"background_tasks_changed:empty","frame":{"type":"system","subtype":"background_tasks_changed","tasks":[],"uuid":"3464e6d2-1a25-4e37-82de-d305440ed44b","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}}
12
{"type":"emit_inbound","label":"task_notification:local_bash","frame":{"type":"system","subtype":"task_notification","task_id":"bc9gkn8ei","tool_use_id":"toolu_01DAnwwvVvLM1cTfzrm6kkor","status":"completed","output_file":"/tmp/claude-replay-claude_background_task_after_root/tasks/bc9gkn8ei.output","summary":"Background command \"sleep 25 && echo L2_BG_DONE\" completed (exit code 0)","uuid":"0a8bc186-10a1-421c-a319-e3c7db9cb4a6","session_id":"ec24a006-617b-4324-b516-57a7afff15a4"}}
13
+{"type":"emit_inbound","label":"assistant:wake-final","frame":{"type":"assistant","message":{"model":"claude-sonnet-4-6","id":"msg_background_command_wake","type":"message","role":"assistant","content":[{"type":"text","text":"L2_WAKE: L2_BG_DONE"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"service_tier":"standard","inference_geo":"not_available"},"context_management":null},"parent_tool_use_id":null,"session_id":"ec24a006-617b-4324-b516-57a7afff15a4","uuid":"39c3f18d-5ec3-48ab-b309-2119f5efb6c5"}}
14
+{"type":"emit_inbound","label":"result:background-wake:1","frame":{"type":"result","subtype":"success","is_error":false,"api_error_status":null,"duration_ms":7125,"duration_api_ms":6500,"num_turns":1,"result":"L2_WAKE: L2_BG_DONE","stop_reason":"end_turn","session_id":"ec24a006-617b-4324-b516-57a7afff15a4","total_cost_usd":0.001,"usage":{"input_tokens":2,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":9,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":0},"inference_geo":"","iterations":[{"input_tokens":2,"output_tokens":9,"cache_read_input_tokens":0,"cache_creation_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"modelUsage":{"claude-sonnet-4-6":{"inputTokens":2,"outputTokens":9,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.001,"contextWindow":200000,"maxOutputTokens":32000}},"permission_denials":[],"terminal_reason":"completed","fast_mode_state":"off","uuid":"43458caa-c0da-459a-a325-af6d22bd0916","origin":{"kind":"task-notification"}}}
15
{"type":"runtime_exit","status":"success"}
apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/input.ts
+9
-3
@@ -1,3 +1,4 @@
1
+import { claudeBackgroundWakeResultLabel } from "../../../Adapters/ClaudeAdapterV2.testkit.ts";
2
import type { OrchestratorFixtureInput } from "../shared.ts";
3
4
export const CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT = [
@@ -16,11 +17,16 @@ export function claudeBackgroundTaskAfterRootInput(): OrchestratorFixtureInput {
17
return {
18
steps: [
19
{ type: "message", text: CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT },
20
+ { type: "await_run_status", targetRunIndex: 1, status: "completed" },
21
+ { type: "release_replay_gate", label: "background_tasks_changed:empty" },
22
{
20
- type: "release_replay_gate_after_waiting",
21
- label: "background_tasks_changed:empty",
22
- targetRunIndex: 1,
23
+ type: "await_run_status",
24
+ targetRunIndex: 2,
25
+ status: "running",
26
+ waitForTurnItemType: "assistant_message",
27
},
28
+ { type: "release_replay_gate", label: claudeBackgroundWakeResultLabel(1) },
29
+ { type: "await_run_status", targetRunIndex: 2, status: "completed" },
30
],
31
};
32
}
apps/server/src/orchestration-v2/testkit/fixtures/claude_background_task_after_root/output.ts
+14
-9
@@ -19,8 +19,8 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
19
assertBaseProjection({
20
result,
21
transcript,
22
- runCount: 1,
23
- runStatuses: ["completed"],
22
+ runCount: 2,
23
+ runStatuses: ["completed", "completed"],
24
});
25
26
const projection = projectionFor(result, transcript.scenario);
@@ -28,7 +28,9 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
28
assertUserMessagesInclude(projection, [CLAUDE_BACKGROUND_TASK_AFTER_ROOT_PROMPT]);
29
30
const rootRun = projection.runs[0];
31
+ const wakeRun = projection.runs[1];
32
assert.isDefined(rootRun);
33
+ assert.isDefined(wakeRun);
34
const pendingRosterIndex = result.domainEvents.findIndex(
35
(event) =>
36
event.type === "provider-thread.updated" &&
@@ -47,12 +49,14 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
49
event.payload.kind === "root_turn" &&
50
event.payload.status === "waiting",
51
);
50
- const idleRosterIndex = result.domainEvents.findIndex(
52
+ const clearedRosterIndex = result.domainEvents.findIndex(
53
(event, index) =>
54
index > waitingRunIndex &&
55
event.type === "provider-thread.updated" &&
54
- event.payload.status === "idle" &&
55
- (event.payload.pendingBackgroundTasks?.length ?? 0) === 0,
56
+ event.payload.pendingBackgroundTasks?.length === 0,
57
+ );
58
+ const wakeRunIndex = result.domainEvents.findIndex(
59
+ (event) => event.type === "run.created" && event.payload.id === wakeRun.id,
60
);
61
62
assert.isAtLeast(pendingRosterIndex, 0, "replay must project the live background task roster");
@@ -66,10 +70,11 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
70
pendingRosterIndex,
71
"checkpoint-waiting root node must retain the background roster",
72
);
73
+ assert.isAbove(wakeRunIndex, waitingRunIndex, "the wake must follow the root turn");
74
assert.isAbove(
70
- idleRosterIndex,
71
- waitingRunIndex,
72
- "late background completion must clear the roster and return the provider thread to idle",
75
+ clearedRosterIndex,
76
+ wakeRunIndex,
77
+ "the background task must remain pending until its wake run consumes the completion",
78
);
79
assert.equal(rootRun.status, "completed");
80
const rootNode = projection.nodes.find(
@@ -86,7 +91,7 @@ export function assertClaudeBackgroundTaskAfterRootOutput(
91
const assistantTexts = projection.turnItems.flatMap((item) =>
92
item.type === "assistant_message" ? [item.text] : [],
93
);
89
- assert.deepEqual(assistantTexts, ["L2_STARTED"]);
94
+ assert.deepEqual(assistantTexts, ["L2_STARTED", "L2_WAKE: L2_BG_DONE"]);
95
96
const shell = result.shellSnapshot.threads.find((thread) => thread.id === projection.thread.id);
97
assert.isDefined(shell);
apps/server/src/orchestration-v2/testkit/fixtures/index.ts
+1
@@ -287,6 +287,7 @@ export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray<OrchestratorReplayFixtu
287
import.meta.url,
288
),
289
modelSelection: CLAUDE_MODEL_SELECTION,
290
+ runContinuationWorker: true,
291
assertOutput: assertClaudeBackgroundTaskAfterRootOutput,
292
},
293
],