@devries-cameron20 / t3code Mirror · read-only / commits / 135d7a9801
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 08:30 UTC

fix(server): a background task's result still shows when the turn before fails (#38035)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Aditya Kumar Puri committed Nov 17, 2026 at 15:23 UTC 135d7a980120790f9b0611a58bae5d9e9feed6f6
2 files changed +171 -10
apps/server/src/orchestration-v2/Orchestrator.ts
+30 -10
@@ -2363,7 +2363,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2363 return true;
2364 });
2365
2366 - const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")(
2366 + const startNextQueuedRunOnce = Effect.fn("Orchestrator.startNextQueuedRun")(
2367 function* (
2368 threadId: ThreadId,
2369 options?: {
@@ -2483,17 +2483,29 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2483 projectOverride !== undefined
2484 ? !projectOverride
2485 : machineInstance !== undefined && !resolveProviderInstanceEnabled(machineInstance);
2486 - if (
2487 - worktreeMissing ||
2488 - instanceDisabledForProject ||
2489 - (failureClass !== undefined &&
2490 - failureClass !== "validation_error" &&
2491 - failedRun?.providerInstanceId === queuedRun.providerInstanceId)
2492 - ) {
2486 + const providerFailed =
2487 + failureClass !== undefined &&
2488 + failureClass !== "validation_error" &&
2489 + failedRun?.providerInstanceId === queuedRun.providerInstanceId;
2490 + if (worktreeMissing || instanceDisabledForProject || providerFailed) {
2491 + // A buffered wake prompts nothing: it replays output the CLI already produced.
2492 + const wakes =
2493 + worktreeMissing || instanceDisabledForProject
2494 + ? []
2495 + : queuedRunsInDeliveryOrder(projection).filter((run) => {
2496 + const message = projection.messages.find(
2497 + (candidate) => candidate.id === run.userMessageId,
2498 + );
2499 + return (
2500 + run.queueHeld !== true &&
2501 + message !== undefined &&
2502 + isBufferedProviderNotification(message)
2503 + );
2504 + });
2505 const now = yield* DateTime.now;
2506 yield* writeSystemEvents(
2507 projection.runs
2496 - .filter((run) => run.status === "queued")
2508 + .filter((run) => run.status === "queued" && !wakes.some((wake) => wake.id === run.id))
2509 .map((run) => ({
2510 type: "run.updated" as const,
2511 threadId,
@@ -2503,7 +2515,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2515 payload: { ...run, queueHeld: true },
2516 })),
2517 );
2506 - return;
2518 + return wakes.length > 0;
2519 }
2520 // Move queued ordinals after runs that started while they waited to preserve transcript and checkpoint order.
2521 // Started runs have prompts; cancelled queued messages do not.
@@ -3235,6 +3247,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3247 ),
3248 );
3249
3250 + // A provider failure hold leaves buffered wakes unheld, so a second pass starts the first one.
3251 + const startNextQueuedRun = (...args: Parameters<typeof startNextQueuedRunOnce>) =>
3252 + startNextQueuedRunOnce(...args).pipe(
3253 + Effect.flatMap((wakeLeftUnheld) =>
3254 + wakeLeftUnheld === true ? startNextQueuedRunOnce(args[0]) : Effect.void,
3255 + ),
3256 + );
3257 +
3258 // Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
3259 const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
3260 threadId: ThreadId,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+141
@@ -10524,6 +10524,147 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
10524 }),
10525 );
10526
10527 + it.effect("starts a provider wake queued behind a run that fails, holding the rest", () =>
10528 + Effect.gen(function* () {
10529 + const orchestrator = yield* Orchestrator.OrchestratorV2;
10530 + const eventSink = yield* EventSink.EventSinkV2;
10531 + const threadId = ThreadId.make("queue-hold-provider-wake-after-failure");
10532 + const messageId = (key: string) => MessageId.make(`${threadId}:${key}`);
10533 + yield* orchestrator.dispatch({
10534 + type: "thread.create",
10535 + createdBy: "user",
10536 + creationSource: "web",
10537 + commandId: CommandId.make(`${threadId}:create`),
10538 + threadId,
10539 + projectId: ProjectId.make(`${threadId}:project`),
10540 + title: "Wake behind a failed run",
10541 + modelSelection,
10542 + runtimeMode: "full-access",
10543 + interactionMode: "default",
10544 + branch: null,
10545 + worktreePath: process.cwd(),
10546 + });
10547 + yield* orchestrator.dispatch({
10548 + type: "message.dispatch",
10549 + createdBy: "user",
10550 + creationSource: "web",
10551 + commandId: CommandId.make(`${threadId}:message:active`),
10552 + threadId,
10553 + messageId: messageId("active"),
10554 + text: "Active",
10555 + attachments: [],
10556 + modelSelection,
10557 + dispatchMode: { type: "start_immediately" },
10558 + });
10559 + yield* orchestrator.dispatch({
10560 + type: "message.dispatch",
10561 + createdBy: "user",
10562 + creationSource: "web",
10563 + commandId: CommandId.make(`${threadId}:message:queued`),
10564 + threadId,
10565 + messageId: messageId("queued"),
10566 + text: "Queued by the user",
10567 + attachments: [],
10568 + modelSelection,
10569 + dispatchMode: { type: "queue_after_active" },
10570 + });
10571 + // The CLI started this execution itself as the active turn ended.
10572 + yield* orchestrator.dispatch({
10573 + type: "message.dispatch",
10574 + createdBy: "agent",
10575 + creationSource: "provider",
10576 + commandId: CommandId.make(`${threadId}:message:wake`),
10577 + threadId,
10578 + messageId: messageId("wake"),
10579 + text: "Background task completed.",
10580 + notification: {
10581 + source: { kind: "background_task" },
10582 + outcome: "updated",
10583 + summary: "Background activity updated",
10584 + },
10585 + attachments: [],
10586 + modelSelection,
10587 + dispatchMode: { type: "queue_after_active" },
10588 + });
10589 + const { runs } = yield* orchestrator.getThreadProjection(threadId);
10590 + const runFor = (key: string) => runs.find((run) => run.userMessageId === messageId(key))!;
10591 + const [active, wake, queued] = [runFor("active"), runFor("wake"), runFor("queued")];
10592 + assert.equal(wake.status, "queued");
10593 + assert.notEqual(wake.queueHeld, true);
10594 + const afterSequence = yield* orchestrator.getThreadEventSequence(threadId);
10595 + const firstUpdateOf = (runId: RunId) =>
10596 + eventSink.stream({ threadId, afterSequence }).pipe(
10597 + Stream.filter(
10598 + (stored) => stored.event.type === "run.updated" && stored.event.payload.id === runId,
10599 + ),
10600 + Stream.runHead,
10601 + Effect.forkScoped,
10602 + );
10603 + const wakeUpdate = yield* firstUpdateOf(wake.id);
10604 + const queuedUpdate = yield* firstUpdateOf(queued.id);
10605 + const now = yield* DateTime.now;
10606 + yield* eventSink.write({
10607 + events: [
10608 + {
10609 + id: EventId.make(`${threadId}:active:error`),
10610 + type: "turn-item.updated",
10611 + threadId,
10612 + runId: active.id,
10613 + nodeId: active.rootNodeId!,
10614 + providerInstanceId: active.providerInstanceId,
10615 + occurredAt: now,
10616 + payload: {
10617 + id: TurnItemId.make(`${threadId}:active:error`),
10618 + type: "error",
10619 + threadId,
10620 + runId: active.id,
10621 + nodeId: active.rootNodeId,
10622 + providerThreadId: active.providerThreadId,
10623 + providerTurnId: null,
10624 + nativeItemRef: null,
10625 + parentItemId: null,
10626 + ordinal: 2,
10627 + status: "failed",
10628 + title: "Provider failure",
10629 + startedAt: now,
10630 + completedAt: now,
10631 + updatedAt: now,
10632 + failure: {
10633 + class: "provider_error",
10634 + message: "Provider request failed with HTTP 502",
10635 + code: null,
10636 + retryable: null,
10637 + },
10638 + },
10639 + },
10640 + {
10641 + id: EventId.make(`${threadId}:active:failed`),
10642 + type: "run.updated",
10643 + threadId,
10644 + runId: active.id,
10645 + providerInstanceId: active.providerInstanceId,
10646 + occurredAt: now,
10647 + payload: { ...active, status: "failed", startedAt: now, completedAt: now },
10648 + },
10649 + ],
10650 + });
10651 + const queuedEvent = Option.getOrUndefined(yield* Fiber.join(queuedUpdate))?.event;
10652 + assert.deepInclude(
10653 + queuedEvent?.type === "run.updated" ? queuedEvent.payload : undefined,
10654 + { status: "queued", queueHeld: true },
10655 + "the user's queued message waits after the provider failure",
10656 + );
10657 + const wakeEvent = Option.getOrUndefined(yield* Fiber.join(wakeUpdate))?.event;
10658 + const wakeRun = wakeEvent?.type === "run.updated" ? wakeEvent.payload : undefined;
10659 + assert.notEqual(
10660 + wakeRun?.queueHeld,
10661 + true,
10662 + "the provider wake replays output the CLI already produced, so the failure must not hold it",
10663 + );
10664 + assert.equal(wakeRun?.status, "starting");
10665 + }),
10666 + );
10667 +
10668 it.effect("edits and removes queued runs", () =>
10669 Effect.gen(function* () {
10670 const orchestrator = yield* Orchestrator.OrchestratorV2;