The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 08:30 UTC
fix(server): a background task's result still shows when the turn before fails (#38035)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Aditya Kumar Puri committed
Nov 17, 2026 at 15:23 UTC
135d7a980120790f9b0611a58bae5d9e9feed6f6
2 files changed
+171
-10
apps/server/src/orchestration-v2/Orchestrator.ts
+30
-10
@@ -2363,7 +2363,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2363
return true;
2364
});
2365
2366
- const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")(
2366
+ const startNextQueuedRunOnce = Effect.fn("Orchestrator.startNextQueuedRun")(
2367
function* (
2368
threadId: ThreadId,
2369
options?: {
@@ -2483,17 +2483,29 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2483
projectOverride !== undefined
2484
? !projectOverride
2485
: machineInstance !== undefined && !resolveProviderInstanceEnabled(machineInstance);
2486
- if (
2487
- worktreeMissing ||
2488
- instanceDisabledForProject ||
2489
- (failureClass !== undefined &&
2490
- failureClass !== "validation_error" &&
2491
- failedRun?.providerInstanceId === queuedRun.providerInstanceId)
2492
- ) {
2486
+ const providerFailed =
2487
+ failureClass !== undefined &&
2488
+ failureClass !== "validation_error" &&
2489
+ failedRun?.providerInstanceId === queuedRun.providerInstanceId;
2490
+ if (worktreeMissing || instanceDisabledForProject || providerFailed) {
2491
+ // A buffered wake prompts nothing: it replays output the CLI already produced.
2492
+ const wakes =
2493
+ worktreeMissing || instanceDisabledForProject
2494
+ ? []
2495
+ : queuedRunsInDeliveryOrder(projection).filter((run) => {
2496
+ const message = projection.messages.find(
2497
+ (candidate) => candidate.id === run.userMessageId,
2498
+ );
2499
+ return (
2500
+ run.queueHeld !== true &&
2501
+ message !== undefined &&
2502
+ isBufferedProviderNotification(message)
2503
+ );
2504
+ });
2505
const now = yield* DateTime.now;
2506
yield* writeSystemEvents(
2507
projection.runs
2496
- .filter((run) => run.status === "queued")
2508
+ .filter((run) => run.status === "queued" && !wakes.some((wake) => wake.id === run.id))
2509
.map((run) => ({
2510
type: "run.updated" as const,
2511
threadId,
@@ -2503,7 +2515,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2515
payload: { ...run, queueHeld: true },
2516
})),
2517
);
2506
- return;
2518
+ return wakes.length > 0;
2519
}
2520
// Move queued ordinals after runs that started while they waited to preserve transcript and checkpoint order.
2521
// Started runs have prompts; cancelled queued messages do not.
@@ -3235,6 +3247,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3247
),
3248
);
3249
3250
+ // A provider failure hold leaves buffered wakes unheld, so a second pass starts the first one.
3251
+ const startNextQueuedRun = (...args: Parameters<typeof startNextQueuedRunOnce>) =>
3252
+ startNextQueuedRunOnce(...args).pipe(
3253
+ Effect.flatMap((wakeLeftUnheld) =>
3254
+ wakeLeftUnheld === true ? startNextQueuedRunOnce(args[0]) : Effect.void,
3255
+ ),
3256
+ );
3257
+
3258
// Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
3259
const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
3260
threadId: ThreadId,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+141
@@ -10524,6 +10524,147 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
10524
}),
10525
);
10526
10527
+ it.effect("starts a provider wake queued behind a run that fails, holding the rest", () =>
10528
+ Effect.gen(function* () {
10529
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
10530
+ const eventSink = yield* EventSink.EventSinkV2;
10531
+ const threadId = ThreadId.make("queue-hold-provider-wake-after-failure");
10532
+ const messageId = (key: string) => MessageId.make(`${threadId}:${key}`);
10533
+ yield* orchestrator.dispatch({
10534
+ type: "thread.create",
10535
+ createdBy: "user",
10536
+ creationSource: "web",
10537
+ commandId: CommandId.make(`${threadId}:create`),
10538
+ threadId,
10539
+ projectId: ProjectId.make(`${threadId}:project`),
10540
+ title: "Wake behind a failed run",
10541
+ modelSelection,
10542
+ runtimeMode: "full-access",
10543
+ interactionMode: "default",
10544
+ branch: null,
10545
+ worktreePath: process.cwd(),
10546
+ });
10547
+ yield* orchestrator.dispatch({
10548
+ type: "message.dispatch",
10549
+ createdBy: "user",
10550
+ creationSource: "web",
10551
+ commandId: CommandId.make(`${threadId}:message:active`),
10552
+ threadId,
10553
+ messageId: messageId("active"),
10554
+ text: "Active",
10555
+ attachments: [],
10556
+ modelSelection,
10557
+ dispatchMode: { type: "start_immediately" },
10558
+ });
10559
+ yield* orchestrator.dispatch({
10560
+ type: "message.dispatch",
10561
+ createdBy: "user",
10562
+ creationSource: "web",
10563
+ commandId: CommandId.make(`${threadId}:message:queued`),
10564
+ threadId,
10565
+ messageId: messageId("queued"),
10566
+ text: "Queued by the user",
10567
+ attachments: [],
10568
+ modelSelection,
10569
+ dispatchMode: { type: "queue_after_active" },
10570
+ });
10571
+ // The CLI started this execution itself as the active turn ended.
10572
+ yield* orchestrator.dispatch({
10573
+ type: "message.dispatch",
10574
+ createdBy: "agent",
10575
+ creationSource: "provider",
10576
+ commandId: CommandId.make(`${threadId}:message:wake`),
10577
+ threadId,
10578
+ messageId: messageId("wake"),
10579
+ text: "Background task completed.",
10580
+ notification: {
10581
+ source: { kind: "background_task" },
10582
+ outcome: "updated",
10583
+ summary: "Background activity updated",
10584
+ },
10585
+ attachments: [],
10586
+ modelSelection,
10587
+ dispatchMode: { type: "queue_after_active" },
10588
+ });
10589
+ const { runs } = yield* orchestrator.getThreadProjection(threadId);
10590
+ const runFor = (key: string) => runs.find((run) => run.userMessageId === messageId(key))!;
10591
+ const [active, wake, queued] = [runFor("active"), runFor("wake"), runFor("queued")];
10592
+ assert.equal(wake.status, "queued");
10593
+ assert.notEqual(wake.queueHeld, true);
10594
+ const afterSequence = yield* orchestrator.getThreadEventSequence(threadId);
10595
+ const firstUpdateOf = (runId: RunId) =>
10596
+ eventSink.stream({ threadId, afterSequence }).pipe(
10597
+ Stream.filter(
10598
+ (stored) => stored.event.type === "run.updated" && stored.event.payload.id === runId,
10599
+ ),
10600
+ Stream.runHead,
10601
+ Effect.forkScoped,
10602
+ );
10603
+ const wakeUpdate = yield* firstUpdateOf(wake.id);
10604
+ const queuedUpdate = yield* firstUpdateOf(queued.id);
10605
+ const now = yield* DateTime.now;
10606
+ yield* eventSink.write({
10607
+ events: [
10608
+ {
10609
+ id: EventId.make(`${threadId}:active:error`),
10610
+ type: "turn-item.updated",
10611
+ threadId,
10612
+ runId: active.id,
10613
+ nodeId: active.rootNodeId!,
10614
+ providerInstanceId: active.providerInstanceId,
10615
+ occurredAt: now,
10616
+ payload: {
10617
+ id: TurnItemId.make(`${threadId}:active:error`),
10618
+ type: "error",
10619
+ threadId,
10620
+ runId: active.id,
10621
+ nodeId: active.rootNodeId,
10622
+ providerThreadId: active.providerThreadId,
10623
+ providerTurnId: null,
10624
+ nativeItemRef: null,
10625
+ parentItemId: null,
10626
+ ordinal: 2,
10627
+ status: "failed",
10628
+ title: "Provider failure",
10629
+ startedAt: now,
10630
+ completedAt: now,
10631
+ updatedAt: now,
10632
+ failure: {
10633
+ class: "provider_error",
10634
+ message: "Provider request failed with HTTP 502",
10635
+ code: null,
10636
+ retryable: null,
10637
+ },
10638
+ },
10639
+ },
10640
+ {
10641
+ id: EventId.make(`${threadId}:active:failed`),
10642
+ type: "run.updated",
10643
+ threadId,
10644
+ runId: active.id,
10645
+ providerInstanceId: active.providerInstanceId,
10646
+ occurredAt: now,
10647
+ payload: { ...active, status: "failed", startedAt: now, completedAt: now },
10648
+ },
10649
+ ],
10650
+ });
10651
+ const queuedEvent = Option.getOrUndefined(yield* Fiber.join(queuedUpdate))?.event;
10652
+ assert.deepInclude(
10653
+ queuedEvent?.type === "run.updated" ? queuedEvent.payload : undefined,
10654
+ { status: "queued", queueHeld: true },
10655
+ "the user's queued message waits after the provider failure",
10656
+ );
10657
+ const wakeEvent = Option.getOrUndefined(yield* Fiber.join(wakeUpdate))?.event;
10658
+ const wakeRun = wakeEvent?.type === "run.updated" ? wakeEvent.payload : undefined;
10659
+ assert.notEqual(
10660
+ wakeRun?.queueHeld,
10661
+ true,
10662
+ "the provider wake replays output the CLI already produced, so the failure must not hold it",
10663
+ );
10664
+ assert.equal(wakeRun?.status, "starting");
10665
+ }),
10666
+ );
10667
+
10668
it.effect("edits and removes queued runs", () =>
10669
Effect.gen(function* () {
10670
const orchestrator = yield* Orchestrator.OrchestratorV2;