@devries-cameron20 / t3code-1 / commits / 3605d52842
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): background tasks survive a failed Claude turn on a live process (#31840)

Julius Marminge committed Nov 16, 2026 at 17:35 UTC 3605d52842c00f728f16978470e73e27c99d231e
4 files changed +234 -35
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+136 -3
@@ -13229,6 +13229,132 @@ describe("ClaudeAdapterV2 background wake turns", () => {
13229 ),
13230 );
13231
13232 + // An API error fails the turn, but the CLI process and its background tasks keep running.
13233 + const liveProcessFailedResult = makeResultFrame({
13234 + uuid: "00000000-0000-4000-8000-000000000311",
13235 + result: "API Error: 502 Bad Gateway",
13236 + isError: true,
13237 + apiErrorStatus: 502,
13238 + terminalReason: "api_error",
13239 + });
13240 +
13241 + it.effect("keeps background tasks waitable after a turn fails on a live process", () =>
13242 + Effect.gen(function* () {
13243 + const harness = yield* makeWakeHarness;
13244 + const awaitBackgroundTasks = harness.runtime.awaitBackgroundTasks;
13245 + if (awaitBackgroundTasks === undefined) {
13246 + throw new Error("Claude adapter runtime must expose awaitBackgroundTasks.");
13247 + }
13248 + yield* harness.runtime.startTurn(
13249 + makeClaudeTestTurnInput({
13250 + threadId: harness.threadId,
13251 + providerThread: harness.providerThread,
13252 + now: yield* DateTime.now,
13253 + attemptId: RunAttemptId.make("attempt-claude-failed-live-roster"),
13254 + text: "Run the build in the background.",
13255 + attachments: [],
13256 + }),
13257 + );
13258 + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
13259 + yield* Queue.offer(harness.sdkMessages, liveProcessFailedResult);
13260 + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed");
13261 +
13262 + assert.deepStrictEqual(
13263 + providerThreadRosterEvents(harness.events)
13264 + .at(-1)
13265 + ?.providerThread.pendingBackgroundTasks?.map((task) => task.taskId),
13266 + [WAKE_TASK_ID],
13267 + );
13268 + assert.isTrue(yield* harness.hasPendingBackgroundWork);
13269 + const marked = yield* awaitBackgroundTasks({
13270 + providerThread: harness.providerThread,
13271 + taskIds: [WAKE_TASK_ID],
13272 + awaited: true,
13273 + });
13274 + assert.deepStrictEqual(marked.matched, [WAKE_TASK_ID]);
13275 + }).pipe(
13276 + Effect.provide(
13277 + Layer.mergeAll(
13278 + IdAllocator.layer,
13279 + McpProviderSessions.layer,
13280 + NodeServices.layer,
13281 + NodeCrypto.layer,
13282 + ),
13283 + ),
13284 + ),
13285 + );
13286 +
13287 + it.effect("clears background tasks when the Claude process exits mid-turn", () =>
13288 + Effect.gen(function* () {
13289 + const harness = yield* makeWakeHarness;
13290 + yield* harness.runtime.startTurn(
13291 + makeClaudeTestTurnInput({
13292 + threadId: harness.threadId,
13293 + providerThread: harness.providerThread,
13294 + now: yield* DateTime.now,
13295 + attemptId: RunAttemptId.make("attempt-claude-probe-exit"),
13296 + text: "Run the build in the background.",
13297 + attachments: [],
13298 + }),
13299 + );
13300 + yield* harness.offerAndWait(wakeTaskStarted);
13301 + assert.isTrue(yield* harness.hasPendingBackgroundWork);
13302 + yield* Queue.shutdown(harness.sdkMessages);
13303 + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed");
13304 + assert.deepStrictEqual(
13305 + providerThreadRosterEvents(harness.events)
13306 + .at(-1)
13307 + ?.providerThread.pendingBackgroundTasks?.map((task) => task.taskId) ?? [],
13308 + [],
13309 + );
13310 + assert.isFalse(yield* harness.hasPendingBackgroundWork);
13311 + }).pipe(
13312 + Effect.provide(
13313 + Layer.mergeAll(
13314 + IdAllocator.layer,
13315 + McpProviderSessions.layer,
13316 + NodeServices.layer,
13317 + NodeCrypto.layer,
13318 + ),
13319 + ),
13320 + ),
13321 + );
13322 +
13323 + it.effect("wakes the idle thread when a task outlives a turn that failed on a live process", () =>
13324 + Effect.gen(function* () {
13325 + const harness = yield* makeWakeHarness;
13326 + yield* harness.runtime.startTurn(
13327 + makeClaudeTestTurnInput({
13328 + threadId: harness.threadId,
13329 + providerThread: harness.providerThread,
13330 + now: yield* DateTime.now,
13331 + attemptId: RunAttemptId.make("attempt-claude-failed-live-wake"),
13332 + text: "Run the build in the background.",
13333 + attachments: [],
13334 + }),
13335 + );
13336 + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
13337 + yield* Queue.offer(harness.sdkMessages, liveProcessFailedResult);
13338 + assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed");
13339 +
13340 + yield* harness.offerAndWait(wakeNotification);
13341 + yield* harness.offerAndWait(wakeAssistant);
13342 + assert.deepStrictEqual(
13343 + harness.continuationRequests.map((request) => request.detail),
13344 + [WAKE_SUMMARY],
13345 + );
13346 + }).pipe(
13347 + Effect.provide(
13348 + Layer.mergeAll(
13349 + IdAllocator.layer,
13350 + McpProviderSessions.layer,
13351 + NodeServices.layer,
13352 + NodeCrypto.layer,
13353 + ),
13354 + ),
13355 + ),
13356 + );
13357 +
13358 it.effect("ScheduleWakeup stop:true releases the pending wakeup's hold", () =>
13359 Effect.gen(function* () {
13360 const harness = yield* makeWakeHarness;
@@ -14435,10 +14561,17 @@ describe("ClaudeAdapterV2 background wake turns", () => {
14561 const rosterBAfterFail = providerThreadRosterEvents(events).findLast(
14562 (event) => event.providerThread.id === providerThreadB.id,
14563 )?.providerThread.pendingBackgroundTasks;
14438 - assert.deepEqual(rosterBAfterFail ?? [], []);
14564 + assert.deepEqual(rosterBAfterFail ?? [], [
14565 + {
14566 + taskId: taskB,
14567 + description: "work on B",
14568 + kind: "command",
14569 + nativeItemId: "toolu-roster-b",
14570 + },
14571 + ]);
14572 assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA));
14440 - assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadB));
14441 - assert.isFalse(yield* hasPendingBackgroundWork);
14573 + assert.isTrue(yield* hasPendingBackgroundWorkForThread(providerThreadB));
14574 + assert.isTrue(yield* hasPendingBackgroundWork);
14575 }).pipe(
14576 Effect.provide(
14577 Layer.mergeAll(
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+9 -3
@@ -9471,11 +9471,18 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9471 Effect.gen(function* () {
9472 const nativeThreadId =
9473 input.context.input.providerThread.nativeThreadRef?.nativeId ?? null;
9474 + const liveQuery = yield* Ref.get(queryContext);
9475 + // Background tasks outlive a failed turn while the process that ran it stays open.
9476 + const processKeepsTasks =
9477 + input.status === "failed" &&
9478 + liveQuery !== null &&
9479 + !liveQuery.stopping &&
9480 + liveQuery.lastTurn === input.context;
9481 if (nativeThreadId !== null) {
9475 - if (input.status !== "completed") {
9482 + if (input.status !== "completed" && !processKeepsTasks) {
9483 // Stop cancels Claude's wakeup and an exited process loses it; a failed
9484 // turn on a live process leaves it pending. Process exits null the query first.
9478 - if (input.status !== "failed" || (yield* Ref.get(queryContext)) === null) {
9485 + if (input.status !== "failed" || liveQuery === null) {
9486 scheduledWakeupsByNativeThread.delete(nativeThreadId);
9487 }
9488 yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId);
@@ -9515,7 +9522,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9522 nativeThreadId === null ? undefined : goalsByNativeThread.get(nativeThreadId);
9523 const goalChecked = goalCheckedTurns.delete(input.context.providerTurnId);
9524 const terminalReason = input.result?.terminal_reason;
9518 - const liveQuery = yield* Ref.get(queryContext);
9525 if (
9526 nativeThreadId !== null &&
9527 goal?.status === "active" &&
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+63 -9
@@ -10,6 +10,7 @@ import {
10 type OrchestrationV2CheckpointScope,
11 type OrchestrationV2DomainEvent,
12 type OrchestrationV2ExecutionNode,
13 + type OrchestrationV2PendingBackgroundTask,
14 type OrchestrationV2ProviderThread,
15 type OrchestrationV2ProviderTurn,
16 type OrchestrationV2Run,
@@ -2393,15 +2394,17 @@ it.effect("keeps ingesting a failed root's background item when its work continu
2394 }),
2395 );
2396
2396 -it.effect("does not pin ingestion on background items when the root turn is interrupted", () =>
2397 - Effect.gen(function* () {
2398 - const observed = yield* runBackgroundItemScenario("bg-interrupted", (ids) => [
2399 - backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2400 - rootTerminalEvent(ids, "interrupted"),
2401 - backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
2402 - ]);
2403 - assert.deepEqual(observed, ["turn_item:running", "root-finalized"]);
2404 - }),
2397 +it.effect.each(["interrupted", "failed"] as const)(
2398 + "does not pin ingestion on background items when the root turn is %s",
2399 + (status) =>
2400 + Effect.gen(function* () {
2401 + const observed = yield* runBackgroundItemScenario(`bg-${status}`, (ids) => [
2402 + backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2403 + rootTerminalEvent(ids, status),
2404 + backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
2405 + ]);
2406 + assert.deepEqual(observed, ["turn_item:running", "root-finalized"]);
2407 + }),
2408 );
2409
2410 it.effect("seeds inherited background items before their next update", () =>
@@ -3927,6 +3930,57 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
3930 }),
3931 );
3932
3933 +it.effect.each(["completed", "failed"] as const)(
3934 + "persists a background-task roster clear that arrives after a %s turn",
3935 + (status) =>
3936 + Effect.gen(function* () {
3937 + const pending = yield* Ref.make(true);
3938 + const ingested = yield* Ref.make<Array<ProviderAdapter.ProviderAdapterV2Event>>([]);
3939 + yield* captureRootRunTermination({
3940 + key: `roster-clear-${status}`,
3941 + shouldFinalizeRun: () => Effect.succeed(true),
3942 + hasPendingBackgroundWorkForThread: () => Ref.get(pending),
3943 + events: (ids) => {
3944 + const runningBuild: OrchestrationV2PendingBackgroundTask = {
3945 + taskId: "running-build",
3946 + kind: "command",
3947 + };
3948 + const roster = (hasTask: boolean): ProviderAdapter.ProviderAdapterV2Event => {
3949 + const pendingBackgroundTasks: ReadonlyArray<OrchestrationV2PendingBackgroundTask> =
3950 + hasTask ? [runningBuild] : [];
3951 + return {
3952 + type: "provider_thread.updated",
3953 + driver,
3954 + providerThread: {
3955 + id: ids.providerThreadId,
3956 + driver,
3957 + appThreadId: ids.threadId,
3958 + pendingBackgroundTasks,
3959 + } as OrchestrationV2ProviderThread,
3960 + };
3961 + };
3962 + return Stream.fromIterable([roster(true), rootTerminalEvent(ids, status), roster(false)]);
3963 + },
3964 + // The adapter clears its live roster before ingestion reaches the queued clear.
3965 + beforeTerminalWrite: () => Ref.set(pending, false).pipe(Effect.as(true)),
3966 + ingestNormalized: (input) =>
3967 + Ref.update(ingested, (all) => [...all, input.event]).pipe(
3968 + Effect.as(
3969 + input.event.type === "provider_thread.updated"
3970 + ? [{} as OrchestrationV2StoredEvent]
3971 + : [],
3972 + ),
3973 + ),
3974 + });
3975 + const rosters = (yield* Ref.get(ingested)).flatMap((event) =>
3976 + event.type === "provider_thread.updated"
3977 + ? [event.providerThread.pendingBackgroundTasks]
3978 + : [],
3979 + );
3980 + assert.deepEqual(rosters.at(-1), [], "an ended task must leave the persisted roster");
3981 + }),
3982 +);
3983 +
3984 function captureRootRunTermination(input: {
3985 readonly key: string;
3986 readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
apps/server/src/orchestration-v2/RunExecutionService.ts
+26 -20
@@ -1249,7 +1249,7 @@ export const layer: Layer.Layer<
1249 const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1250 // Open tool calls on the root thread, to end from their stored rows if the runtime is lost.
1251 const openRootToolCalls = new Set<TurnItemId>();
1252 - // The last stored root snapshot's pendingBackgroundWork; adapters that omit it leave it false.
1252 + // The last stored root snapshot still reports background work, by flag or task roster.
1253 let rootSnapshotPendingWork = false;
1254 // Owned child threads whose last snapshot reported pending work, such as a queued native
1255 // continuation. Unlike the root's, these must hold the stream past owner loss.
@@ -1614,6 +1614,21 @@ export const layer: Layer.Layer<
1614 const childSubagents = yield* Ref.get(activeChildSubagents);
1615 return childSubagents.size === 0;
1616 });
1617 + // Claude background Bash has no turn-item projection. Use only the
1618 + // thread-scoped probe: session-wide pending work (siblings, wake
1619 + // buffers, session subagents) must not pin this root subscription.
1620 + const threadHasPendingBackgroundWork = Effect.gen(function* () {
1621 + // The live probe reads adapter state, which can be ahead of the events consumed here.
1622 + if (rootSnapshotPendingWork) {
1623 + return true;
1624 + }
1625 + if (input.session.hasPendingBackgroundWorkForThread === undefined) {
1626 + return false;
1627 + }
1628 + return yield* input.session
1629 + .hasPendingBackgroundWorkForThread(yield* Ref.get(latestProviderThread))
1630 + .pipe(Effect.catchCause(() => Effect.succeed(false)));
1631 + });
1632 const shouldStopProviderEventIngestion = Effect.gen(function* () {
1633 if (!(yield* agentTurnsSettled)) {
1634 return false;
@@ -1624,7 +1639,12 @@ export const layer: Layer.Layer<
1639 terminal.status !== "completed" &&
1640 !childTurnsContinue(terminal)
1641 ) {
1627 - return true;
1642 + // A failed turn can leave background work running in a live provider process.
1643 + return (
1644 + terminal.status !== "failed" ||
1645 + (yield* Ref.get(providerThreadOwnerLost)) ||
1646 + !(yield* threadHasPendingBackgroundWork)
1647 + );
1648 }
1649 // Newly owned items depend on adapters emitting a non-terminal event before the root
1650 // terminal. Owner loss (a newer run claimed lastRunOrdinal) must not close this stream
@@ -1637,23 +1657,7 @@ export const layer: Layer.Layer<
1657 if (yield* Ref.get(providerThreadOwnerLost)) {
1658 return true;
1659 }
1640 - // The live probe reads adapter state, which can be ahead of the events consumed here.
1641 - if (rootSnapshotPendingWork) {
1642 - return false;
1643 - }
1644 - // Claude background Bash has no turn-item projection. Use only the
1645 - // thread-scoped probe: session-wide pending work (siblings, wake
1646 - // buffers, session subagents) must not pin this root subscription.
1647 - const latestProviderThreadSnapshot = yield* Ref.get(latestProviderThread);
1648 - if (input.session.hasPendingBackgroundWorkForThread !== undefined) {
1649 - const hasPendingWork = yield* input.session
1650 - .hasPendingBackgroundWorkForThread(latestProviderThreadSnapshot)
1651 - .pipe(Effect.catchCause(() => Effect.succeed(false)));
1652 - if (hasPendingWork) {
1653 - return false;
1654 - }
1655 - }
1656 - return true;
1660 + return !(yield* threadHasPendingBackgroundWork);
1661 });
1662 // Held text writes nothing, so its item would take a position only
1663 // when finally written, behind every item that began after it.
@@ -1934,7 +1938,9 @@ export const layer: Layer.Layer<
1938 if (event.type === "provider_thread.updated") {
1939 if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) {
1940 yield* Ref.set(latestProviderThread, event.providerThread);
1937 - rootSnapshotPendingWork = event.pendingBackgroundWork === true;
1941 + rootSnapshotPendingWork =
1942 + event.pendingBackgroundWork === true ||
1943 + (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0;
1944 } else if (event.providerThread.id !== input.providerThread.id) {
1945 if (event.pendingBackgroundWork === true) {
1946 childSnapshotPendingWork.add(event.providerThread.id);