The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 09:45 UTC
fix(server): background tasks survive a failed Claude turn on a live process (#31840)
Julius Marminge committed
Nov 16, 2026 at 17:35 UTC
3605d52842c00f728f16978470e73e27c99d231e
4 files changed
+234
-35
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+136
-3
@@ -13229,6 +13229,132 @@ describe("ClaudeAdapterV2 background wake turns", () => {
13229
),
13230
);
13231
13232
+ // An API error fails the turn, but the CLI process and its background tasks keep running.
13233
+ const liveProcessFailedResult = makeResultFrame({
13234
+ uuid: "00000000-0000-4000-8000-000000000311",
13235
+ result: "API Error: 502 Bad Gateway",
13236
+ isError: true,
13237
+ apiErrorStatus: 502,
13238
+ terminalReason: "api_error",
13239
+ });
13240
+
13241
+ it.effect("keeps background tasks waitable after a turn fails on a live process", () =>
13242
+ Effect.gen(function* () {
13243
+ const harness = yield* makeWakeHarness;
13244
+ const awaitBackgroundTasks = harness.runtime.awaitBackgroundTasks;
13245
+ if (awaitBackgroundTasks === undefined) {
13246
+ throw new Error("Claude adapter runtime must expose awaitBackgroundTasks.");
13247
+ }
13248
+ yield* harness.runtime.startTurn(
13249
+ makeClaudeTestTurnInput({
13250
+ threadId: harness.threadId,
13251
+ providerThread: harness.providerThread,
13252
+ now: yield* DateTime.now,
13253
+ attemptId: RunAttemptId.make("attempt-claude-failed-live-roster"),
13254
+ text: "Run the build in the background.",
13255
+ attachments: [],
13256
+ }),
13257
+ );
13258
+ yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
13259
+ yield* Queue.offer(harness.sdkMessages, liveProcessFailedResult);
13260
+ assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed");
13261
+
13262
+ assert.deepStrictEqual(
13263
+ providerThreadRosterEvents(harness.events)
13264
+ .at(-1)
13265
+ ?.providerThread.pendingBackgroundTasks?.map((task) => task.taskId),
13266
+ [WAKE_TASK_ID],
13267
+ );
13268
+ assert.isTrue(yield* harness.hasPendingBackgroundWork);
13269
+ const marked = yield* awaitBackgroundTasks({
13270
+ providerThread: harness.providerThread,
13271
+ taskIds: [WAKE_TASK_ID],
13272
+ awaited: true,
13273
+ });
13274
+ assert.deepStrictEqual(marked.matched, [WAKE_TASK_ID]);
13275
+ }).pipe(
13276
+ Effect.provide(
13277
+ Layer.mergeAll(
13278
+ IdAllocator.layer,
13279
+ McpProviderSessions.layer,
13280
+ NodeServices.layer,
13281
+ NodeCrypto.layer,
13282
+ ),
13283
+ ),
13284
+ ),
13285
+ );
13286
+
13287
+ it.effect("clears background tasks when the Claude process exits mid-turn", () =>
13288
+ Effect.gen(function* () {
13289
+ const harness = yield* makeWakeHarness;
13290
+ yield* harness.runtime.startTurn(
13291
+ makeClaudeTestTurnInput({
13292
+ threadId: harness.threadId,
13293
+ providerThread: harness.providerThread,
13294
+ now: yield* DateTime.now,
13295
+ attemptId: RunAttemptId.make("attempt-claude-probe-exit"),
13296
+ text: "Run the build in the background.",
13297
+ attachments: [],
13298
+ }),
13299
+ );
13300
+ yield* harness.offerAndWait(wakeTaskStarted);
13301
+ assert.isTrue(yield* harness.hasPendingBackgroundWork);
13302
+ yield* Queue.shutdown(harness.sdkMessages);
13303
+ assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed");
13304
+ assert.deepStrictEqual(
13305
+ providerThreadRosterEvents(harness.events)
13306
+ .at(-1)
13307
+ ?.providerThread.pendingBackgroundTasks?.map((task) => task.taskId) ?? [],
13308
+ [],
13309
+ );
13310
+ assert.isFalse(yield* harness.hasPendingBackgroundWork);
13311
+ }).pipe(
13312
+ Effect.provide(
13313
+ Layer.mergeAll(
13314
+ IdAllocator.layer,
13315
+ McpProviderSessions.layer,
13316
+ NodeServices.layer,
13317
+ NodeCrypto.layer,
13318
+ ),
13319
+ ),
13320
+ ),
13321
+ );
13322
+
13323
+ it.effect("wakes the idle thread when a task outlives a turn that failed on a live process", () =>
13324
+ Effect.gen(function* () {
13325
+ const harness = yield* makeWakeHarness;
13326
+ yield* harness.runtime.startTurn(
13327
+ makeClaudeTestTurnInput({
13328
+ threadId: harness.threadId,
13329
+ providerThread: harness.providerThread,
13330
+ now: yield* DateTime.now,
13331
+ attemptId: RunAttemptId.make("attempt-claude-failed-live-wake"),
13332
+ text: "Run the build in the background.",
13333
+ attachments: [],
13334
+ }),
13335
+ );
13336
+ yield* Queue.offer(harness.sdkMessages, wakeTaskStarted);
13337
+ yield* Queue.offer(harness.sdkMessages, liveProcessFailedResult);
13338
+ assert.equal((yield* Queue.take(harness.terminalReceipts)).status, "failed");
13339
+
13340
+ yield* harness.offerAndWait(wakeNotification);
13341
+ yield* harness.offerAndWait(wakeAssistant);
13342
+ assert.deepStrictEqual(
13343
+ harness.continuationRequests.map((request) => request.detail),
13344
+ [WAKE_SUMMARY],
13345
+ );
13346
+ }).pipe(
13347
+ Effect.provide(
13348
+ Layer.mergeAll(
13349
+ IdAllocator.layer,
13350
+ McpProviderSessions.layer,
13351
+ NodeServices.layer,
13352
+ NodeCrypto.layer,
13353
+ ),
13354
+ ),
13355
+ ),
13356
+ );
13357
+
13358
it.effect("ScheduleWakeup stop:true releases the pending wakeup's hold", () =>
13359
Effect.gen(function* () {
13360
const harness = yield* makeWakeHarness;
@@ -14435,10 +14561,17 @@ describe("ClaudeAdapterV2 background wake turns", () => {
14561
const rosterBAfterFail = providerThreadRosterEvents(events).findLast(
14562
(event) => event.providerThread.id === providerThreadB.id,
14563
)?.providerThread.pendingBackgroundTasks;
14438
- assert.deepEqual(rosterBAfterFail ?? [], []);
14564
+ assert.deepEqual(rosterBAfterFail ?? [], [
14565
+ {
14566
+ taskId: taskB,
14567
+ description: "work on B",
14568
+ kind: "command",
14569
+ nativeItemId: "toolu-roster-b",
14570
+ },
14571
+ ]);
14572
assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadA));
14440
- assert.isFalse(yield* hasPendingBackgroundWorkForThread(providerThreadB));
14441
- assert.isFalse(yield* hasPendingBackgroundWork);
14573
+ assert.isTrue(yield* hasPendingBackgroundWorkForThread(providerThreadB));
14574
+ assert.isTrue(yield* hasPendingBackgroundWork);
14575
}).pipe(
14576
Effect.provide(
14577
Layer.mergeAll(
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+9
-3
@@ -9471,11 +9471,18 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9471
Effect.gen(function* () {
9472
const nativeThreadId =
9473
input.context.input.providerThread.nativeThreadRef?.nativeId ?? null;
9474
+ const liveQuery = yield* Ref.get(queryContext);
9475
+ // Background tasks outlive a failed turn while the process that ran it stays open.
9476
+ const processKeepsTasks =
9477
+ input.status === "failed" &&
9478
+ liveQuery !== null &&
9479
+ !liveQuery.stopping &&
9480
+ liveQuery.lastTurn === input.context;
9481
if (nativeThreadId !== null) {
9475
- if (input.status !== "completed") {
9482
+ if (input.status !== "completed" && !processKeepsTasks) {
9483
// Stop cancels Claude's wakeup and an exited process loses it; a failed
9484
// turn on a live process leaves it pending. Process exits null the query first.
9478
- if (input.status !== "failed" || (yield* Ref.get(queryContext)) === null) {
9485
+ if (input.status !== "failed" || liveQuery === null) {
9486
scheduledWakeupsByNativeThread.delete(nativeThreadId);
9487
}
9488
yield* clearPendingBackgroundTasksForNativeThread(nativeThreadId);
@@ -9515,7 +9522,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9522
nativeThreadId === null ? undefined : goalsByNativeThread.get(nativeThreadId);
9523
const goalChecked = goalCheckedTurns.delete(input.context.providerTurnId);
9524
const terminalReason = input.result?.terminal_reason;
9518
- const liveQuery = yield* Ref.get(queryContext);
9525
if (
9526
nativeThreadId !== null &&
9527
goal?.status === "active" &&
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+63
-9
@@ -10,6 +10,7 @@ import {
10
type OrchestrationV2CheckpointScope,
11
type OrchestrationV2DomainEvent,
12
type OrchestrationV2ExecutionNode,
13
+ type OrchestrationV2PendingBackgroundTask,
14
type OrchestrationV2ProviderThread,
15
type OrchestrationV2ProviderTurn,
16
type OrchestrationV2Run,
@@ -2393,15 +2394,17 @@ it.effect("keeps ingesting a failed root's background item when its work continu
2394
}),
2395
);
2396
2396
-it.effect("does not pin ingestion on background items when the root turn is interrupted", () =>
2397
- Effect.gen(function* () {
2398
- const observed = yield* runBackgroundItemScenario("bg-interrupted", (ids) => [
2399
- backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2400
- rootTerminalEvent(ids, "interrupted"),
2401
- backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
2402
- ]);
2403
- assert.deepEqual(observed, ["turn_item:running", "root-finalized"]);
2404
- }),
2397
+it.effect.each(["interrupted", "failed"] as const)(
2398
+ "does not pin ingestion on background items when the root turn is %s",
2399
+ (status) =>
2400
+ Effect.gen(function* () {
2401
+ const observed = yield* runBackgroundItemScenario(`bg-${status}`, (ids) => [
2402
+ backgroundTurnItemEvent(ids, "command_execution", "running", 1),
2403
+ rootTerminalEvent(ids, status),
2404
+ backgroundTurnItemEvent(ids, "command_execution", "completed", 2),
2405
+ ]);
2406
+ assert.deepEqual(observed, ["turn_item:running", "root-finalized"]);
2407
+ }),
2408
);
2409
2410
it.effect("seeds inherited background items before their next update", () =>
@@ -3927,6 +3930,57 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
3930
}),
3931
);
3932
3933
+it.effect.each(["completed", "failed"] as const)(
3934
+ "persists a background-task roster clear that arrives after a %s turn",
3935
+ (status) =>
3936
+ Effect.gen(function* () {
3937
+ const pending = yield* Ref.make(true);
3938
+ const ingested = yield* Ref.make<Array<ProviderAdapter.ProviderAdapterV2Event>>([]);
3939
+ yield* captureRootRunTermination({
3940
+ key: `roster-clear-${status}`,
3941
+ shouldFinalizeRun: () => Effect.succeed(true),
3942
+ hasPendingBackgroundWorkForThread: () => Ref.get(pending),
3943
+ events: (ids) => {
3944
+ const runningBuild: OrchestrationV2PendingBackgroundTask = {
3945
+ taskId: "running-build",
3946
+ kind: "command",
3947
+ };
3948
+ const roster = (hasTask: boolean): ProviderAdapter.ProviderAdapterV2Event => {
3949
+ const pendingBackgroundTasks: ReadonlyArray<OrchestrationV2PendingBackgroundTask> =
3950
+ hasTask ? [runningBuild] : [];
3951
+ return {
3952
+ type: "provider_thread.updated",
3953
+ driver,
3954
+ providerThread: {
3955
+ id: ids.providerThreadId,
3956
+ driver,
3957
+ appThreadId: ids.threadId,
3958
+ pendingBackgroundTasks,
3959
+ } as OrchestrationV2ProviderThread,
3960
+ };
3961
+ };
3962
+ return Stream.fromIterable([roster(true), rootTerminalEvent(ids, status), roster(false)]);
3963
+ },
3964
+ // The adapter clears its live roster before ingestion reaches the queued clear.
3965
+ beforeTerminalWrite: () => Ref.set(pending, false).pipe(Effect.as(true)),
3966
+ ingestNormalized: (input) =>
3967
+ Ref.update(ingested, (all) => [...all, input.event]).pipe(
3968
+ Effect.as(
3969
+ input.event.type === "provider_thread.updated"
3970
+ ? [{} as OrchestrationV2StoredEvent]
3971
+ : [],
3972
+ ),
3973
+ ),
3974
+ });
3975
+ const rosters = (yield* Ref.get(ingested)).flatMap((event) =>
3976
+ event.type === "provider_thread.updated"
3977
+ ? [event.providerThread.pendingBackgroundTasks]
3978
+ : [],
3979
+ );
3980
+ assert.deepEqual(rosters.at(-1), [], "an ended task must leave the persisted roster");
3981
+ }),
3982
+);
3983
+
3984
function captureRootRunTermination(input: {
3985
readonly key: string;
3986
readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
apps/server/src/orchestration-v2/RunExecutionService.ts
+26
-20
@@ -1249,7 +1249,7 @@ export const layer: Layer.Layer<
1249
const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1250
// Open tool calls on the root thread, to end from their stored rows if the runtime is lost.
1251
const openRootToolCalls = new Set<TurnItemId>();
1252
- // The last stored root snapshot's pendingBackgroundWork; adapters that omit it leave it false.
1252
+ // The last stored root snapshot still reports background work, by flag or task roster.
1253
let rootSnapshotPendingWork = false;
1254
// Owned child threads whose last snapshot reported pending work, such as a queued native
1255
// continuation. Unlike the root's, these must hold the stream past owner loss.
@@ -1614,6 +1614,21 @@ export const layer: Layer.Layer<
1614
const childSubagents = yield* Ref.get(activeChildSubagents);
1615
return childSubagents.size === 0;
1616
});
1617
+ // Claude background Bash has no turn-item projection. Use only the
1618
+ // thread-scoped probe: session-wide pending work (siblings, wake
1619
+ // buffers, session subagents) must not pin this root subscription.
1620
+ const threadHasPendingBackgroundWork = Effect.gen(function* () {
1621
+ // The live probe reads adapter state, which can be ahead of the events consumed here.
1622
+ if (rootSnapshotPendingWork) {
1623
+ return true;
1624
+ }
1625
+ if (input.session.hasPendingBackgroundWorkForThread === undefined) {
1626
+ return false;
1627
+ }
1628
+ return yield* input.session
1629
+ .hasPendingBackgroundWorkForThread(yield* Ref.get(latestProviderThread))
1630
+ .pipe(Effect.catchCause(() => Effect.succeed(false)));
1631
+ });
1632
const shouldStopProviderEventIngestion = Effect.gen(function* () {
1633
if (!(yield* agentTurnsSettled)) {
1634
return false;
@@ -1624,7 +1639,12 @@ export const layer: Layer.Layer<
1639
terminal.status !== "completed" &&
1640
!childTurnsContinue(terminal)
1641
) {
1627
- return true;
1642
+ // A failed turn can leave background work running in a live provider process.
1643
+ return (
1644
+ terminal.status !== "failed" ||
1645
+ (yield* Ref.get(providerThreadOwnerLost)) ||
1646
+ !(yield* threadHasPendingBackgroundWork)
1647
+ );
1648
}
1649
// Newly owned items depend on adapters emitting a non-terminal event before the root
1650
// terminal. Owner loss (a newer run claimed lastRunOrdinal) must not close this stream
@@ -1637,23 +1657,7 @@ export const layer: Layer.Layer<
1657
if (yield* Ref.get(providerThreadOwnerLost)) {
1658
return true;
1659
}
1640
- // The live probe reads adapter state, which can be ahead of the events consumed here.
1641
- if (rootSnapshotPendingWork) {
1642
- return false;
1643
- }
1644
- // Claude background Bash has no turn-item projection. Use only the
1645
- // thread-scoped probe: session-wide pending work (siblings, wake
1646
- // buffers, session subagents) must not pin this root subscription.
1647
- const latestProviderThreadSnapshot = yield* Ref.get(latestProviderThread);
1648
- if (input.session.hasPendingBackgroundWorkForThread !== undefined) {
1649
- const hasPendingWork = yield* input.session
1650
- .hasPendingBackgroundWorkForThread(latestProviderThreadSnapshot)
1651
- .pipe(Effect.catchCause(() => Effect.succeed(false)));
1652
- if (hasPendingWork) {
1653
- return false;
1654
- }
1655
- }
1656
- return true;
1660
+ return !(yield* threadHasPendingBackgroundWork);
1661
});
1662
// Held text writes nothing, so its item would take a position only
1663
// when finally written, behind every item that began after it.
@@ -1934,7 +1938,9 @@ export const layer: Layer.Layer<
1938
if (event.type === "provider_thread.updated") {
1939
if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) {
1940
yield* Ref.set(latestProviderThread, event.providerThread);
1937
- rootSnapshotPendingWork = event.pendingBackgroundWork === true;
1941
+ rootSnapshotPendingWork =
1942
+ event.pendingBackgroundWork === true ||
1943
+ (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0;
1944
} else if (event.providerThread.id !== input.providerThread.id) {
1945
if (event.pendingBackgroundWork === true) {
1946
childSnapshotPendingWork.add(event.providerThread.id);