@devries-cameron20 / t3code Mirror · read-only / commits / 6f6f68ad2d
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 19:30 UTC

fix(server): stop abandoned turns before continuing threads (#23169)

Yash Singh committed Oct 21, 2026 at 20:47 UTC 6f6f68ad2d8044b3322ec738b9d371fcaaa783cf
6 files changed +489 -29
apps/mobile/src/features/threads/worktree-setup-card.tsx
+2 -2
@@ -84,7 +84,7 @@ export function WorktreeSetupCard(props: WorktreeSetupCardProps) {
84 accessibilityRole="button"
85 accessibilityLabel={
86 backgroundSetup
87 - ? `${scriptName} is still running. Show setup progress.`
87 + ? `Background setup is still running: ${scriptName}. Show setup progress.`
88 : "Worktree setup details"
89 }
90 accessibilityState={{ expanded: detailsOpen }}
@@ -116,7 +116,7 @@ export function WorktreeSetupCard(props: WorktreeSetupCardProps) {
116 />
117 ) : null}
118 <Text numberOfLines={1} className="shrink text-2xs text-foreground-secondary">
119 - {backgroundSetup ? scriptName : "Details"}
119 + {backgroundSetup ? "Background setup" : "Details"}
120 </Text>
121 {!backgroundSetup ? (
122 <SymbolView name="chevron.right" size={10} tintColorClassName="accent-icon-muted" />
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+197 -4
@@ -7,6 +7,7 @@ import {
7 NodeId,
8 ProviderSessionId,
9 ProviderThreadId,
10 + ProviderTurnId,
11 ProviderDriverKind,
12 ProviderInstanceId,
13 ProviderSetupError,
@@ -271,6 +272,14 @@ function makeLocalCommandHarness(input: {
272 readonly interruptRunBeforeOpenFailure?: boolean;
273 readonly writeFailure?: unknown;
274 readonly failReadsAfterRunning?: boolean;
275 + readonly previousTurn?: {
276 + readonly runStatus: "failed" | "completed";
277 + readonly turnStatus: "running" | "completed" | "interrupted";
278 + readonly differentThread?: boolean;
279 + readonly liveSession?: boolean;
280 + readonly interruptFailure?: boolean;
281 + readonly interruptCurrentRun?: boolean;
282 + };
283 }) {
284 const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z");
285 const threadId = ThreadId.make("thread-native-account-command");
@@ -456,6 +465,74 @@ function makeLocalCommandHarness(input: {
465 ),
466 };
467 }
468 + const previousTurnId = ProviderTurnId.make("previous-provider-turn");
469 + if (input.previousTurn !== undefined) {
470 + const previousRunId = RunId.make("previous-failed-run");
471 + const previousAttemptId = RunAttemptId.make("previous-failed-attempt");
472 + const previousProviderThreadId = input.previousTurn.differentThread
473 + ? oldProviderThreadId
474 + : providerThreadId;
475 + projection = {
476 + ...projection,
477 + runs: [
478 + {
479 + ...run,
480 + id: previousRunId,
481 + ordinal: 1,
482 + status: input.previousTurn.runStatus,
483 + rootNodeId: NodeId.make("previous-root"),
484 + activeAttemptId: previousAttemptId,
485 + providerThreadId: previousProviderThreadId,
486 + completedAt: now,
487 + },
488 + run,
489 + ],
490 + providerThreads: [
491 + {
492 + ...providerThread,
493 + nativeThreadRef: {
494 + driver: providerThread.driver,
495 + nativeId: "existing-native-conversation",
496 + strength: "strong",
497 + },
498 + },
499 + ],
500 + providerTurns: [
501 + {
502 + id: previousTurnId,
503 + providerThreadId: previousProviderThreadId,
504 + nodeId: NodeId.make("previous-root"),
505 + runAttemptId: previousAttemptId,
506 + ordinal: 1,
507 + nativeTurnRef: null,
508 + status: input.previousTurn.turnStatus,
509 + startedAt: now,
510 + completedAt: input.previousTurn.turnStatus === "running" ? null : now,
511 + },
512 + ],
513 + };
514 + }
515 + let providerBusy =
516 + input.previousTurn?.runStatus === "failed" &&
517 + input.previousTurn.turnStatus === "running" &&
518 + !input.previousTurn.differentThread &&
519 + input.previousTurn.liveSession !== false;
520 + const interruptTurn = vi.fn((request: ProviderAdapter.ProviderAdapterV2InterruptInput) =>
521 + Effect.gen(function* () {
522 + expect(request.providerTurnId).toBe(previousTurnId);
523 + expect(request.providerThread.id).toBe(providerThreadId);
524 + if (input.previousTurn?.interruptFailure) {
525 + return yield* new ProviderAdapter.ProviderAdapterInterruptError({
526 + driver: providerThread.driver,
527 + providerThreadId,
528 + providerTurnId: request.providerTurnId,
529 + cause: "provider interrupt failed",
530 + });
531 + }
532 + providerBusy = false;
533 + if (input.previousTurn?.interruptCurrentRun) interruptRun();
534 + }),
535 + );
536 const events: Array<OrchestrationV2DomainEvent> = [];
537 const interruptRun = () => {
538 projection = {
@@ -515,7 +592,7 @@ function makeLocalCommandHarness(input: {
592 ),
593 ),
594 )
518 - : input.failReadsAfterRunning === true
595 + : input.failReadsAfterRunning === true || input.previousTurn !== undefined
596 ? Effect.succeed({
597 driver: providerThread.driver,
598 providerSession: {
@@ -531,14 +608,17 @@ function makeLocalCommandHarness(input: {
608 lastError: null,
609 },
610 ensureThread: () => Effect.succeed(providerThread),
611 + resumeThread: () => Effect.succeed(projection.providerThreads[0]),
612 } as never)
613 : Effect.die("A local command must not open a native session."),
614 );
615 const startRootRun = vi.fn<
616 (input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect<void>
617 >(() =>
540 - input.failReadsAfterRunning === true
541 - ? Effect.void
618 + input.failReadsAfterRunning === true || input.previousTurn !== undefined
619 + ? Effect.suspend(() =>
620 + providerBusy ? Effect.die("The previous provider turn is still running.") : Effect.void,
621 + )
622 : Effect.die("A local command must not start a native turn."),
623 );
624 const failReadIfRunning = Effect.suspend(() =>
@@ -642,7 +722,19 @@ function makeLocalCommandHarness(input: {
722 }),
723 ),
724 }),
645 - Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open }),
725 + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
726 + open,
727 + get: () =>
728 + Effect.succeed(
729 + input.previousTurn?.liveSession === false
730 + ? Option.none()
731 + : Option.some({
732 + driver: providerThread.driver,
733 + providerSessionId,
734 + interruptTurn,
735 + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime),
736 + ),
737 + }),
738 Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),
739 Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),
740 Layer.mock(RuntimePolicy.RuntimePolicyV2)({
@@ -662,6 +754,7 @@ function makeLocalCommandHarness(input: {
754 open,
755 writeIfRunCurrent,
756 startRootRun,
757 + interruptTurn,
758 tryHandlePromptCommand,
759 events,
760 oldInstanceId,
@@ -681,6 +774,106 @@ function makeLocalCommandHarness(input: {
774 };
775 }
776
777 +effectIt.effect("recovers an abandoned failed turn before continuing its native conversation", () =>
778 + Effect.gen(function* () {
779 + const harness = makeLocalCommandHarness({
780 + text: "Continue",
781 + previousTurn: { runStatus: "failed", turnStatus: "running" },
782 + });
783 +
784 + yield* harness.start;
785 +
786 + expect(harness.interruptTurn).toHaveBeenCalledOnce();
787 + expect(harness.projection().runs.at(-1)?.status).toBe("running");
788 + expect(harness.projection().providerTurns[0]).toMatchObject({
789 + status: "interrupted",
790 + completedAt: expect.anything(),
791 + });
792 + expect(harness.startRootRun).toHaveBeenCalledOnce();
793 + expect(harness.startRootRun.mock.calls[0]?.[0].providerThread.nativeThreadRef?.nativeId).toBe(
794 + "existing-native-conversation",
795 + );
796 + }),
797 +);
798 +
799 +effectIt.effect.each([
800 + { runStatus: "failed", turnStatus: "completed" },
801 + { runStatus: "completed", turnStatus: "completed" },
802 + { runStatus: "failed", turnStatus: "running", differentThread: true },
803 + { runStatus: "failed", turnStatus: "running", liveSession: false },
804 + { runStatus: "failed", turnStatus: "interrupted", interruptFailure: true },
805 +] as const)("continues without interrupting when recovery does not apply: %j", (previousTurn) =>
806 + Effect.gen(function* () {
807 + const harness = makeLocalCommandHarness({ text: "Continue", previousTurn });
808 +
809 + yield* harness.start;
810 +
811 + expect(harness.interruptTurn).not.toHaveBeenCalled();
812 + expect(harness.projection().runs.at(-1)?.status).toBe("running");
813 + }),
814 +);
815 +
816 +effectIt.effect("does not start another turn when abandoned-turn cleanup fails", () =>
817 + Effect.gen(function* () {
818 + const harness = makeLocalCommandHarness({
819 + text: "Continue",
820 + previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
821 + });
822 +
823 + yield* harness.start;
824 +
825 + expect(harness.interruptTurn).toHaveBeenCalledOnce();
826 + expect(harness.startRootRun).not.toHaveBeenCalled();
827 + expect(harness.projection().runs.at(-1)?.status).toBe("failed");
828 + }),
829 +);
830 +
831 +effectIt.effect("retries abandoned-turn cleanup without failing the continuation early", () =>
832 + Effect.gen(function* () {
833 + const harness = makeLocalCommandHarness({
834 + text: "Continue",
835 + previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
836 + });
837 +
838 + const error = yield* harness.startWithRetry.pipe(Effect.flip);
839 +
840 + expect(error._tag).toBe("ProviderTurnStartError");
841 + expect(harness.projection().runs.at(-1)?.status).toBe("starting");
842 + expect(harness.open).not.toHaveBeenCalled();
843 + }),
844 +);
845 +
846 +effectIt.effect("does not start the continuation if recording recovery fails", () =>
847 + Effect.gen(function* () {
848 + const harness = makeLocalCommandHarness({
849 + text: "Continue",
850 + previousTurn: { runStatus: "failed", turnStatus: "running" },
851 + writeFailure: new Error("database unavailable"),
852 + });
853 +
854 + const error = yield* harness.start.pipe(Effect.flip);
855 +
856 + expect(error._tag).toBe("ProviderTurnStartError");
857 + expect(harness.projection().runs.at(-1)?.status).toBe("starting");
858 + expect(harness.projection().providerTurns[0]?.status).toBe("running");
859 + expect(harness.open).not.toHaveBeenCalled();
860 + }),
861 +);
862 +
863 +effectIt.effect("does not resume a continuation stopped while recovery was in flight", () =>
864 + Effect.gen(function* () {
865 + const harness = makeLocalCommandHarness({
866 + text: "Continue",
867 + previousTurn: { runStatus: "failed", turnStatus: "running", interruptCurrentRun: true },
868 + });
869 +
870 + yield* harness.start;
871 +
872 + expect(harness.projection().runs.at(-1)?.status).toBe("interrupted");
873 + expect(harness.open).not.toHaveBeenCalled();
874 + }),
875 +);
876 +
877 effectIt.effect.each(["cleanup-first", "startup-first"])(
878 "coordinates provider startup with %s",
879 (order) =>
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+85 -20
@@ -541,26 +541,6 @@ export const layer: Layer.Layer<
541 const existingSessionProjection = projection.providerSessions.find(
542 (candidate) => candidate.id === providerSessionId,
543 );
544 - const sessionResult = yield* Effect.result(
545 - providerSessions.open({
546 - threadId: projection.thread.id,
547 - providerSessionId,
548 - modelSelection: run.modelSelection,
549 - runtimePolicy: resolvedRuntimePolicy,
550 - ...(existingSessionProjection === undefined
551 - ? {}
552 - : { resumeFromSession: existingSessionProjection }),
553 - ...(providerThread.nativeThreadRef?.nativeId == null
554 - ? {}
555 - : { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }),
556 - ...(providerThread.nativeMetadata?.itemIdentityVersion === undefined
557 - ? {}
558 - : {
559 - initialProviderItemIdentityVersion:
560 - providerThread.nativeMetadata.itemIdentityVersion,
561 - }),
562 - }),
563 - );
544 const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
545 function* (failed: {
546 readonly signal: string;
@@ -592,6 +572,91 @@ export const layer: Layer.Layer<
572 });
573 },
574 );
575 + const failedRuns = new Map(
576 + projection.runs
577 + .filter(
578 + (previous) =>
579 + previous.ordinal < run.ordinal &&
580 + previous.providerThreadId === providerThread.id &&
581 + previous.status === "failed",
582 + )
583 + .map((previous) => [previous.activeAttemptId, previous] as const),
584 + );
585 + const abandonedTurns = projection.providerTurns.flatMap((turn) => {
586 + const previous = failedRuns.get(turn.runAttemptId);
587 + return turn.providerThreadId === providerThread.id &&
588 + turn.status === "running" &&
589 + previous !== undefined &&
590 + turn.nodeId === previous.rootNodeId
591 + ? [{ turn, run: previous }]
592 + : [];
593 + });
594 + const abandonedTurn = abandonedTurns.at(-1)?.turn;
595 + if (abandonedTurn !== undefined) {
596 + const liveSession = yield* providerSessions.get(providerSessionId);
597 + if (Option.isSome(liveSession)) {
598 + // Older versions could fail the run without stopping its provider.
599 + // Recover that exact turn before reusing its native conversation.
600 + const recovered = yield* Effect.result(
601 + liveSession.value.interruptTurn({
602 + providerThread,
603 + providerTurnId: abandonedTurn.id,
604 + requestRuntimeRestart: true,
605 + }),
606 + );
607 + if (recovered._tag === "Failure") {
608 + if (input.willRetry === true) return yield* recovered.failure;
609 + yield* settleStartFailure({
610 + signal: "provider-turn-recovery-failure",
611 + title: "Provider turn recovery failed",
612 + error: recovered.failure,
613 + });
614 + return;
615 + }
616 + }
617 + const completedAt = yield* DateTime.now;
618 + const repaired = yield* eventSink.writeIfRunCurrent({
619 + threadId: projection.thread.id,
620 + runId: run.id,
621 + activeAttemptId: attempt.id,
622 + expectedStatus: "starting",
623 + events: yield* Effect.forEach(abandonedTurns, ({ turn, run: failedRun }) =>
624 + Effect.gen(function* () {
625 + return {
626 + id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
627 + type: "provider-turn.updated" as const,
628 + threadId: projection.thread.id,
629 + runId: failedRun.id,
630 + nodeId: turn.nodeId,
631 + providerInstanceId: failedRun.providerInstanceId,
632 + occurredAt: completedAt,
633 + payload: { ...turn, status: "interrupted" as const, completedAt },
634 + };
635 + }),
636 + ),
637 + });
638 + if (!repaired.committed) return;
639 + }
640 + const sessionResult = yield* Effect.result(
641 + providerSessions.open({
642 + threadId: projection.thread.id,
643 + providerSessionId,
644 + modelSelection: run.modelSelection,
645 + runtimePolicy: resolvedRuntimePolicy,
646 + ...(existingSessionProjection === undefined
647 + ? {}
648 + : { resumeFromSession: existingSessionProjection }),
649 + ...(providerThread.nativeThreadRef?.nativeId == null
650 + ? {}
651 + : { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }),
652 + ...(providerThread.nativeMetadata?.itemIdentityVersion === undefined
653 + ? {}
654 + : {
655 + initialProviderItemIdentityVersion:
656 + providerThread.nativeMetadata.itemIdentityVersion,
657 + }),
658 + }),
659 + );
660 if (sessionResult._tag === "Failure") {
661 if (input.willRetry === true) return yield* sessionResult.failure;
662 yield* settleStartFailure({
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+153 -1
@@ -2487,6 +2487,152 @@ it.effect("refreshes pull requests after a provider stream exits with an error",
2487 }),
2488 );
2489
2490 +it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as const)(
2491 + "stops the live provider and background work before settling an ingestion %s",
2492 + (failureMode) =>
2493 + Effect.gen(function* () {
2494 + const started = yield* Deferred.make<void>();
2495 + const stopped = yield* Deferred.make<void>();
2496 + const backgroundWorkRunning = yield* Ref.make(true);
2497 + const backgroundWorkAtFinalization = yield* Ref.make<boolean | null>(null);
2498 + const now = yield* DateTime.now;
2499 + const interrupts = yield* Ref.make<
2500 + ReadonlyArray<ProviderAdapter.ProviderAdapterV2InterruptInput>
2501 + >([]);
2502 + const ids = backgroundScenarioIds(`ingestion:${failureMode}`);
2503 + const { observed, finalEvents } = yield* captureRootRunTermination({
2504 + key: `ingestion:${failureMode}`,
2505 + initialProviderTurnId: null,
2506 + rejectTerminalWrite: failureMode === "ownership-lost",
2507 + shouldFinalizeRun: () => Effect.succeed(true),
2508 + startTurn: () =>
2509 + Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(stopped))),
2510 + events: () =>
2511 + Stream.unwrap(
2512 + Deferred.await(started).pipe(
2513 + Effect.as(
2514 + Stream.make({
2515 + type: "provider_turn.updated" as const,
2516 + driver,
2517 + providerTurn: {
2518 + id: ids.rootProviderTurnId,
2519 + providerThreadId: ids.providerThreadId,
2520 + nodeId: ids.rootNodeId,
2521 + runAttemptId: ids.attemptId,
2522 + nativeTurnRef: null,
2523 + ordinal: 1,
2524 + status: "running" as const,
2525 + startedAt: now,
2526 + completedAt: null,
2527 + },
2528 + }),
2529 + ),
2530 + ),
2531 + ),
2532 + ingestNormalized: () => {
2533 + const error = new ProviderEventIngestor.ProviderEventPublishError({
2534 + providerSessionId: ProviderSessionId.make(`session:${failureMode}`),
2535 + eventCount: 1,
2536 + cause: new Error("SQLITE_FULL: database or disk is full"),
2537 + });
2538 + return failureMode === "defect" ? Effect.die(error) : Effect.fail(error);
2539 + },
2540 + interruptTurn: (input) =>
2541 + Ref.update(interrupts, (current) => [...current, input]).pipe(
2542 + Effect.andThen(
2543 + input.requestRuntimeRestart === true && failureMode !== "interrupt-failure"
2544 + ? Ref.set(backgroundWorkRunning, false)
2545 + : Effect.void,
2546 + ),
2547 + Effect.andThen(Deferred.succeed(stopped, undefined)),
2548 + Effect.andThen(
2549 + failureMode === "interrupt-failure"
2550 + ? Effect.fail(
2551 + new ProviderAdapter.ProviderAdapterInterruptError({
2552 + driver,
2553 + providerThreadId: input.providerThread.id,
2554 + providerTurnId: input.providerTurnId,
2555 + cause: "provider already exited",
2556 + }),
2557 + )
2558 + : Effect.void,
2559 + ),
2560 + ),
2561 + beforeTerminalWrite: () =>
2562 + Effect.gen(function* () {
2563 + const calls = yield* Ref.get(interrupts);
2564 + assert.equal(calls.length, 1);
2565 + assert.equal(calls[0]?.providerTurnId, ids.rootProviderTurnId);
2566 + assert.equal(calls[0]?.providerThread.id, ids.providerThreadId);
2567 + yield* Ref.set(backgroundWorkAtFinalization, yield* Ref.get(backgroundWorkRunning));
2568 + return true;
2569 + }),
2570 + });
2571 + if (failureMode === "ownership-lost") {
2572 + assert.deepEqual(finalEvents, []);
2573 + assert.deepEqual(observed, []);
2574 + } else {
2575 + assert.equal(
2576 + yield* Ref.get(backgroundWorkAtFinalization),
2577 + failureMode === "interrupt-failure",
2578 + );
2579 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2580 + const repaired = finalEvents.find((event) => event.type === "provider-turn.updated");
2581 + if (failureMode === "interrupt-failure") {
2582 + assert.isUndefined(repaired);
2583 + } else {
2584 + assert.equal(repaired?.payload.id, ids.rootProviderTurnId);
2585 + assert.equal(repaired?.payload.status, "interrupted");
2586 + assert.isNotNull(repaired?.payload.completedAt);
2587 + }
2588 + }
2589 + }),
2590 +);
2591 +
2592 +it.effect("does not interrupt an ended provider when its terminal write fails", () =>
2593 + Effect.gen(function* () {
2594 + const interrupts = yield* Ref.make(0);
2595 + const writes = yield* Ref.make(0);
2596 + const { observed } = yield* captureRootRunTermination({
2597 + key: "terminal-write-failure",
2598 + shouldFinalizeRun: () => Effect.succeed(true),
2599 + events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")),
2600 + interruptTurn: () => Ref.update(interrupts, (count) => count + 1),
2601 + beforeTerminalWrite: () =>
2602 + Effect.gen(function* () {
2603 + if ((yield* Ref.updateAndGet(writes, (count) => count + 1)) === 1) {
2604 + return yield* new EventSink.EventSinkWriteError({ eventCount: 1 });
2605 + }
2606 + return true;
2607 + }),
2608 + });
2609 + assert.equal(yield* Ref.get(interrupts), 0);
2610 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2611 + }),
2612 +);
2613 +
2614 +it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () =>
2615 + Effect.gen(function* () {
2616 + const interrupts = yield* Ref.make(0);
2617 + const { observed } = yield* captureRootRunTermination({
2618 + key: "unknown-turn-stream-error",
2619 + initialProviderTurnId: null,
2620 + shouldFinalizeRun: () => Effect.succeed(true),
2621 + events: () =>
2622 + Stream.fail(
2623 + new ProviderAdapter.ProviderAdapterEventStreamError({
2624 + driver,
2625 + providerSessionId: ProviderSessionId.make("session:unknown-turn"),
2626 + cause: "provider process exited",
2627 + }),
2628 + ),
2629 + interruptTurn: () => Ref.update(interrupts, (count) => count + 1),
2630 + });
2631 + assert.equal(yield* Ref.get(interrupts), 0);
2632 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2633 + }),
2634 +);
2635 +
2636 it.effect("fails the run when its event stream ends without a root terminal", () =>
2637 Effect.gen(function* () {
2638 const { observed, finalEvents } = yield* captureRootRunTermination({
@@ -2564,6 +2710,8 @@ function captureRootRunTermination(input: {
2710 ProviderAdapter.ProviderAdapterV2Error
2711 >;
2712 readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
2713 + readonly interruptTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["interruptTurn"];
2714 + readonly initialProviderTurnId?: ProviderTurnId | null;
2715 readonly onStartFailure?: (
2716 error: RunExecutionService.RunExecutionServiceV2Error,
2717 ) => Effect.Effect<void>;
@@ -2704,6 +2852,7 @@ function captureRootRunTermination(input: {
2852 ),
2853 }),
2854 startTurn: input.startTurn ?? (() => Effect.void),
2855 + interruptTurn: input.interruptTurn ?? (() => Effect.void),
2856 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2857 run: {
2858 id: ids.runId,
@@ -2724,7 +2873,10 @@ function captureRootRunTermination(input: {
2873 } as OrchestrationV2ProviderThread,
2874 attempt: {
2875 id: ids.attemptId,
2727 - providerTurnId: ids.rootProviderTurnId,
2876 + providerTurnId:
2877 + input.initialProviderTurnId === undefined
2878 + ? ids.rootProviderTurnId
2879 + : input.initialProviderTurnId,
2880 } as OrchestrationV2RunAttempt,
2881 attemptId: ids.attemptId,
2882 providerTurnOrdinal: 1,
apps/server/src/orchestration-v2/RunExecutionService.ts
+50
@@ -622,6 +622,7 @@ export const layer: Layer.Layer<
622 readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>;
623 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
624 readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
625 + readonly interruptedProviderTurn?: OrchestrationV2ProviderTurn;
626 readonly terminal: ProviderTerminalEvent;
627 readonly completedAt?: DateTime.Utc;
628 readonly failureItemPersisted: boolean;
@@ -743,6 +744,24 @@ export const layer: Layer.Layer<
744 : [],
745 events: [
746 ...cascadedSubagentEvents,
747 + ...(input.interruptedProviderTurn === undefined
748 + ? []
749 + : [
750 + {
751 + id: yield* allocateEventId(),
752 + type: "provider-turn.updated" as const,
753 + threadId: input.run.threadId,
754 + runId: input.run.id,
755 + nodeId: input.rootNode.id,
756 + providerInstanceId: input.run.providerInstanceId,
757 + occurredAt: completedAt,
758 + payload: {
759 + ...input.interruptedProviderTurn,
760 + status: "interrupted" as const,
761 + completedAt,
762 + },
763 + },
764 + ]),
765 ...(finalizedAttempt === null
766 ? []
767 : [
@@ -962,6 +981,7 @@ export const layer: Layer.Layer<
981 const terminalEvent = yield* Ref.make<ProviderTerminalEvent | null>(null);
982 const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100);
983 const latestProviderThread = yield* Ref.make(input.providerThread);
984 + const latestRootProviderTurn = yield* Ref.make<OrchestrationV2ProviderTurn | null>(null);
985 const routeIdentity: ProviderEventRouteIdentity = {
986 threadId: input.run.threadId,
987 runId: input.run.id,
@@ -1318,6 +1338,13 @@ export const layer: Layer.Layer<
1338 Stream.tap(({ event, accepted }) =>
1339 Effect.gen(function* () {
1340 if (!accepted) return;
1341 + if (
1342 + event.type === "provider_turn.updated" &&
1343 + event.providerTurn.runAttemptId === input.attempt.id &&
1344 + event.providerTurn.providerThreadId === input.providerThread.id
1345 + ) {
1346 + yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1347 + }
1348 let storedEventCount = 0;
1349 const deliveredEvent = filterAssistantEvent(
1350 event,
@@ -1436,6 +1463,28 @@ export const layer: Layer.Layer<
1463 });
1464 if (yield* Ref.get(rootRunFinalized)) return;
1465 const providerThread = yield* Ref.get(latestProviderThread);
1466 + const routing = yield* Ref.get(eventRouting);
1467 + // Stop background work too; the failed consumer cannot persist terminal events.
1468 + const interruptedProviderTurn =
1469 + routing.rootProviderTurnId !== null && !routing.rootTurnEnded
1470 + ? yield* input.session
1471 + .interruptTurn({
1472 + providerThread,
1473 + providerTurnId: routing.rootProviderTurnId,
1474 + requestRuntimeRestart: true,
1475 + })
1476 + .pipe(
1477 + Effect.as(yield* Ref.get(latestRootProviderTurn)),
1478 + Effect.catchCauseIf(
1479 + (interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1480 + (interruptCause) =>
1481 + Effect.logWarning(
1482 + "orchestration V2 failed to interrupt turn after ingestion failure",
1483 + { runId: input.run.id, cause: interruptCause },
1484 + ).pipe(Effect.as(null)),
1485 + ),
1486 + )
1487 + : null;
1488 const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
1489 const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1490 let nextRetryWarningAt = completedAt;
@@ -1450,6 +1499,7 @@ export const layer: Layer.Layer<
1499 expectedStatus: "running",
1500 },
1501 openRunOwnedSubagents: openSubagents,
1502 + ...(interruptedProviderTurn === null ? {} : { interruptedProviderTurn }),
1503 completedAt,
1504 terminal: makeFailedTerminalEvent(
1505 makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),
apps/web/src/components/chat/TimelineWorkGroupRows.tsx
+2 -2
@@ -330,12 +330,12 @@ function BackgroundWorktreeSetupChip({ snapshot }: { snapshot: WorktreeSetupSnap
330 variant="ghost-muted"
331 size="micro"
332 className="ml-auto min-w-0 shrink-0"
333 - aria-label={`${scriptName} is still running. Show setup progress.`}
333 + aria-label={`Background setup is still running: ${scriptName}. Show setup progress.`}
334 />
335 }
336 >
337 <Spinner className="size-3 shrink-0" />
338 - <span className="truncate">{scriptName}</span>
338 + <span className="truncate">Background setup</span>
339 </PopoverTrigger>
340 <PopoverPopup side="bottom" align="end" width="lg" padding="compact">
341 <WorktreeSetupCard