The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 19:30 UTC
fix(server): stop abandoned turns before continuing threads (#23169)
Yash Singh committed
Oct 21, 2026 at 20:47 UTC
6f6f68ad2d8044b3322ec738b9d371fcaaa783cf
6 files changed
+489
-29
apps/mobile/src/features/threads/worktree-setup-card.tsx
+2
-2
@@ -84,7 +84,7 @@ export function WorktreeSetupCard(props: WorktreeSetupCardProps) {
84
accessibilityRole="button"
85
accessibilityLabel={
86
backgroundSetup
87
- ? `${scriptName} is still running. Show setup progress.`
87
+ ? `Background setup is still running: ${scriptName}. Show setup progress.`
88
: "Worktree setup details"
89
}
90
accessibilityState={{ expanded: detailsOpen }}
@@ -116,7 +116,7 @@ export function WorktreeSetupCard(props: WorktreeSetupCardProps) {
116
/>
117
) : null}
118
<Text numberOfLines={1} className="shrink text-2xs text-foreground-secondary">
119
- {backgroundSetup ? scriptName : "Details"}
119
+ {backgroundSetup ? "Background setup" : "Details"}
120
</Text>
121
{!backgroundSetup ? (
122
<SymbolView name="chevron.right" size={10} tintColorClassName="accent-icon-muted" />
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+197
-4
@@ -7,6 +7,7 @@ import {
7
NodeId,
8
ProviderSessionId,
9
ProviderThreadId,
10
+ ProviderTurnId,
11
ProviderDriverKind,
12
ProviderInstanceId,
13
ProviderSetupError,
@@ -271,6 +272,14 @@ function makeLocalCommandHarness(input: {
272
readonly interruptRunBeforeOpenFailure?: boolean;
273
readonly writeFailure?: unknown;
274
readonly failReadsAfterRunning?: boolean;
275
+ readonly previousTurn?: {
276
+ readonly runStatus: "failed" | "completed";
277
+ readonly turnStatus: "running" | "completed" | "interrupted";
278
+ readonly differentThread?: boolean;
279
+ readonly liveSession?: boolean;
280
+ readonly interruptFailure?: boolean;
281
+ readonly interruptCurrentRun?: boolean;
282
+ };
283
}) {
284
const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z");
285
const threadId = ThreadId.make("thread-native-account-command");
@@ -456,6 +465,74 @@ function makeLocalCommandHarness(input: {
465
),
466
};
467
}
468
+ const previousTurnId = ProviderTurnId.make("previous-provider-turn");
469
+ if (input.previousTurn !== undefined) {
470
+ const previousRunId = RunId.make("previous-failed-run");
471
+ const previousAttemptId = RunAttemptId.make("previous-failed-attempt");
472
+ const previousProviderThreadId = input.previousTurn.differentThread
473
+ ? oldProviderThreadId
474
+ : providerThreadId;
475
+ projection = {
476
+ ...projection,
477
+ runs: [
478
+ {
479
+ ...run,
480
+ id: previousRunId,
481
+ ordinal: 1,
482
+ status: input.previousTurn.runStatus,
483
+ rootNodeId: NodeId.make("previous-root"),
484
+ activeAttemptId: previousAttemptId,
485
+ providerThreadId: previousProviderThreadId,
486
+ completedAt: now,
487
+ },
488
+ run,
489
+ ],
490
+ providerThreads: [
491
+ {
492
+ ...providerThread,
493
+ nativeThreadRef: {
494
+ driver: providerThread.driver,
495
+ nativeId: "existing-native-conversation",
496
+ strength: "strong",
497
+ },
498
+ },
499
+ ],
500
+ providerTurns: [
501
+ {
502
+ id: previousTurnId,
503
+ providerThreadId: previousProviderThreadId,
504
+ nodeId: NodeId.make("previous-root"),
505
+ runAttemptId: previousAttemptId,
506
+ ordinal: 1,
507
+ nativeTurnRef: null,
508
+ status: input.previousTurn.turnStatus,
509
+ startedAt: now,
510
+ completedAt: input.previousTurn.turnStatus === "running" ? null : now,
511
+ },
512
+ ],
513
+ };
514
+ }
515
+ let providerBusy =
516
+ input.previousTurn?.runStatus === "failed" &&
517
+ input.previousTurn.turnStatus === "running" &&
518
+ !input.previousTurn.differentThread &&
519
+ input.previousTurn.liveSession !== false;
520
+ const interruptTurn = vi.fn((request: ProviderAdapter.ProviderAdapterV2InterruptInput) =>
521
+ Effect.gen(function* () {
522
+ expect(request.providerTurnId).toBe(previousTurnId);
523
+ expect(request.providerThread.id).toBe(providerThreadId);
524
+ if (input.previousTurn?.interruptFailure) {
525
+ return yield* new ProviderAdapter.ProviderAdapterInterruptError({
526
+ driver: providerThread.driver,
527
+ providerThreadId,
528
+ providerTurnId: request.providerTurnId,
529
+ cause: "provider interrupt failed",
530
+ });
531
+ }
532
+ providerBusy = false;
533
+ if (input.previousTurn?.interruptCurrentRun) interruptRun();
534
+ }),
535
+ );
536
const events: Array<OrchestrationV2DomainEvent> = [];
537
const interruptRun = () => {
538
projection = {
@@ -515,7 +592,7 @@ function makeLocalCommandHarness(input: {
592
),
593
),
594
)
518
- : input.failReadsAfterRunning === true
595
+ : input.failReadsAfterRunning === true || input.previousTurn !== undefined
596
? Effect.succeed({
597
driver: providerThread.driver,
598
providerSession: {
@@ -531,14 +608,17 @@ function makeLocalCommandHarness(input: {
608
lastError: null,
609
},
610
ensureThread: () => Effect.succeed(providerThread),
611
+ resumeThread: () => Effect.succeed(projection.providerThreads[0]),
612
} as never)
613
: Effect.die("A local command must not open a native session."),
614
);
615
const startRootRun = vi.fn<
616
(input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect<void>
617
>(() =>
540
- input.failReadsAfterRunning === true
541
- ? Effect.void
618
+ input.failReadsAfterRunning === true || input.previousTurn !== undefined
619
+ ? Effect.suspend(() =>
620
+ providerBusy ? Effect.die("The previous provider turn is still running.") : Effect.void,
621
+ )
622
: Effect.die("A local command must not start a native turn."),
623
);
624
const failReadIfRunning = Effect.suspend(() =>
@@ -642,7 +722,19 @@ function makeLocalCommandHarness(input: {
722
}),
723
),
724
}),
645
- Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ open }),
725
+ Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
726
+ open,
727
+ get: () =>
728
+ Effect.succeed(
729
+ input.previousTurn?.liveSession === false
730
+ ? Option.none()
731
+ : Option.some({
732
+ driver: providerThread.driver,
733
+ providerSessionId,
734
+ interruptTurn,
735
+ } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime),
736
+ ),
737
+ }),
738
Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),
739
Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),
740
Layer.mock(RuntimePolicy.RuntimePolicyV2)({
@@ -662,6 +754,7 @@ function makeLocalCommandHarness(input: {
754
open,
755
writeIfRunCurrent,
756
startRootRun,
757
+ interruptTurn,
758
tryHandlePromptCommand,
759
events,
760
oldInstanceId,
@@ -681,6 +774,106 @@ function makeLocalCommandHarness(input: {
774
};
775
}
776
777
+effectIt.effect("recovers an abandoned failed turn before continuing its native conversation", () =>
778
+ Effect.gen(function* () {
779
+ const harness = makeLocalCommandHarness({
780
+ text: "Continue",
781
+ previousTurn: { runStatus: "failed", turnStatus: "running" },
782
+ });
783
+
784
+ yield* harness.start;
785
+
786
+ expect(harness.interruptTurn).toHaveBeenCalledOnce();
787
+ expect(harness.projection().runs.at(-1)?.status).toBe("running");
788
+ expect(harness.projection().providerTurns[0]).toMatchObject({
789
+ status: "interrupted",
790
+ completedAt: expect.anything(),
791
+ });
792
+ expect(harness.startRootRun).toHaveBeenCalledOnce();
793
+ expect(harness.startRootRun.mock.calls[0]?.[0].providerThread.nativeThreadRef?.nativeId).toBe(
794
+ "existing-native-conversation",
795
+ );
796
+ }),
797
+);
798
+
799
+effectIt.effect.each([
800
+ { runStatus: "failed", turnStatus: "completed" },
801
+ { runStatus: "completed", turnStatus: "completed" },
802
+ { runStatus: "failed", turnStatus: "running", differentThread: true },
803
+ { runStatus: "failed", turnStatus: "running", liveSession: false },
804
+ { runStatus: "failed", turnStatus: "interrupted", interruptFailure: true },
805
+] as const)("continues without interrupting when recovery does not apply: %j", (previousTurn) =>
806
+ Effect.gen(function* () {
807
+ const harness = makeLocalCommandHarness({ text: "Continue", previousTurn });
808
+
809
+ yield* harness.start;
810
+
811
+ expect(harness.interruptTurn).not.toHaveBeenCalled();
812
+ expect(harness.projection().runs.at(-1)?.status).toBe("running");
813
+ }),
814
+);
815
+
816
+effectIt.effect("does not start another turn when abandoned-turn cleanup fails", () =>
817
+ Effect.gen(function* () {
818
+ const harness = makeLocalCommandHarness({
819
+ text: "Continue",
820
+ previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
821
+ });
822
+
823
+ yield* harness.start;
824
+
825
+ expect(harness.interruptTurn).toHaveBeenCalledOnce();
826
+ expect(harness.startRootRun).not.toHaveBeenCalled();
827
+ expect(harness.projection().runs.at(-1)?.status).toBe("failed");
828
+ }),
829
+);
830
+
831
+effectIt.effect("retries abandoned-turn cleanup without failing the continuation early", () =>
832
+ Effect.gen(function* () {
833
+ const harness = makeLocalCommandHarness({
834
+ text: "Continue",
835
+ previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
836
+ });
837
+
838
+ const error = yield* harness.startWithRetry.pipe(Effect.flip);
839
+
840
+ expect(error._tag).toBe("ProviderTurnStartError");
841
+ expect(harness.projection().runs.at(-1)?.status).toBe("starting");
842
+ expect(harness.open).not.toHaveBeenCalled();
843
+ }),
844
+);
845
+
846
+effectIt.effect("does not start the continuation if recording recovery fails", () =>
847
+ Effect.gen(function* () {
848
+ const harness = makeLocalCommandHarness({
849
+ text: "Continue",
850
+ previousTurn: { runStatus: "failed", turnStatus: "running" },
851
+ writeFailure: new Error("database unavailable"),
852
+ });
853
+
854
+ const error = yield* harness.start.pipe(Effect.flip);
855
+
856
+ expect(error._tag).toBe("ProviderTurnStartError");
857
+ expect(harness.projection().runs.at(-1)?.status).toBe("starting");
858
+ expect(harness.projection().providerTurns[0]?.status).toBe("running");
859
+ expect(harness.open).not.toHaveBeenCalled();
860
+ }),
861
+);
862
+
863
+effectIt.effect("does not resume a continuation stopped while recovery was in flight", () =>
864
+ Effect.gen(function* () {
865
+ const harness = makeLocalCommandHarness({
866
+ text: "Continue",
867
+ previousTurn: { runStatus: "failed", turnStatus: "running", interruptCurrentRun: true },
868
+ });
869
+
870
+ yield* harness.start;
871
+
872
+ expect(harness.projection().runs.at(-1)?.status).toBe("interrupted");
873
+ expect(harness.open).not.toHaveBeenCalled();
874
+ }),
875
+);
876
+
877
effectIt.effect.each(["cleanup-first", "startup-first"])(
878
"coordinates provider startup with %s",
879
(order) =>
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+85
-20
@@ -541,26 +541,6 @@ export const layer: Layer.Layer<
541
const existingSessionProjection = projection.providerSessions.find(
542
(candidate) => candidate.id === providerSessionId,
543
);
544
- const sessionResult = yield* Effect.result(
545
- providerSessions.open({
546
- threadId: projection.thread.id,
547
- providerSessionId,
548
- modelSelection: run.modelSelection,
549
- runtimePolicy: resolvedRuntimePolicy,
550
- ...(existingSessionProjection === undefined
551
- ? {}
552
- : { resumeFromSession: existingSessionProjection }),
553
- ...(providerThread.nativeThreadRef?.nativeId == null
554
- ? {}
555
- : { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }),
556
- ...(providerThread.nativeMetadata?.itemIdentityVersion === undefined
557
- ? {}
558
- : {
559
- initialProviderItemIdentityVersion:
560
- providerThread.nativeMetadata.itemIdentityVersion,
561
- }),
562
- }),
563
- );
544
const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
545
function* (failed: {
546
readonly signal: string;
@@ -592,6 +572,91 @@ export const layer: Layer.Layer<
572
});
573
},
574
);
575
+ const failedRuns = new Map(
576
+ projection.runs
577
+ .filter(
578
+ (previous) =>
579
+ previous.ordinal < run.ordinal &&
580
+ previous.providerThreadId === providerThread.id &&
581
+ previous.status === "failed",
582
+ )
583
+ .map((previous) => [previous.activeAttemptId, previous] as const),
584
+ );
585
+ const abandonedTurns = projection.providerTurns.flatMap((turn) => {
586
+ const previous = failedRuns.get(turn.runAttemptId);
587
+ return turn.providerThreadId === providerThread.id &&
588
+ turn.status === "running" &&
589
+ previous !== undefined &&
590
+ turn.nodeId === previous.rootNodeId
591
+ ? [{ turn, run: previous }]
592
+ : [];
593
+ });
594
+ const abandonedTurn = abandonedTurns.at(-1)?.turn;
595
+ if (abandonedTurn !== undefined) {
596
+ const liveSession = yield* providerSessions.get(providerSessionId);
597
+ if (Option.isSome(liveSession)) {
598
+ // Older versions could fail the run without stopping its provider.
599
+ // Recover that exact turn before reusing its native conversation.
600
+ const recovered = yield* Effect.result(
601
+ liveSession.value.interruptTurn({
602
+ providerThread,
603
+ providerTurnId: abandonedTurn.id,
604
+ requestRuntimeRestart: true,
605
+ }),
606
+ );
607
+ if (recovered._tag === "Failure") {
608
+ if (input.willRetry === true) return yield* recovered.failure;
609
+ yield* settleStartFailure({
610
+ signal: "provider-turn-recovery-failure",
611
+ title: "Provider turn recovery failed",
612
+ error: recovered.failure,
613
+ });
614
+ return;
615
+ }
616
+ }
617
+ const completedAt = yield* DateTime.now;
618
+ const repaired = yield* eventSink.writeIfRunCurrent({
619
+ threadId: projection.thread.id,
620
+ runId: run.id,
621
+ activeAttemptId: attempt.id,
622
+ expectedStatus: "starting",
623
+ events: yield* Effect.forEach(abandonedTurns, ({ turn, run: failedRun }) =>
624
+ Effect.gen(function* () {
625
+ return {
626
+ id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
627
+ type: "provider-turn.updated" as const,
628
+ threadId: projection.thread.id,
629
+ runId: failedRun.id,
630
+ nodeId: turn.nodeId,
631
+ providerInstanceId: failedRun.providerInstanceId,
632
+ occurredAt: completedAt,
633
+ payload: { ...turn, status: "interrupted" as const, completedAt },
634
+ };
635
+ }),
636
+ ),
637
+ });
638
+ if (!repaired.committed) return;
639
+ }
640
+ const sessionResult = yield* Effect.result(
641
+ providerSessions.open({
642
+ threadId: projection.thread.id,
643
+ providerSessionId,
644
+ modelSelection: run.modelSelection,
645
+ runtimePolicy: resolvedRuntimePolicy,
646
+ ...(existingSessionProjection === undefined
647
+ ? {}
648
+ : { resumeFromSession: existingSessionProjection }),
649
+ ...(providerThread.nativeThreadRef?.nativeId == null
650
+ ? {}
651
+ : { initialNativeThreadId: providerThread.nativeThreadRef.nativeId }),
652
+ ...(providerThread.nativeMetadata?.itemIdentityVersion === undefined
653
+ ? {}
654
+ : {
655
+ initialProviderItemIdentityVersion:
656
+ providerThread.nativeMetadata.itemIdentityVersion,
657
+ }),
658
+ }),
659
+ );
660
if (sessionResult._tag === "Failure") {
661
if (input.willRetry === true) return yield* sessionResult.failure;
662
yield* settleStartFailure({
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+153
-1
@@ -2487,6 +2487,152 @@ it.effect("refreshes pull requests after a provider stream exits with an error",
2487
}),
2488
);
2489
2490
+it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as const)(
2491
+ "stops the live provider and background work before settling an ingestion %s",
2492
+ (failureMode) =>
2493
+ Effect.gen(function* () {
2494
+ const started = yield* Deferred.make<void>();
2495
+ const stopped = yield* Deferred.make<void>();
2496
+ const backgroundWorkRunning = yield* Ref.make(true);
2497
+ const backgroundWorkAtFinalization = yield* Ref.make<boolean | null>(null);
2498
+ const now = yield* DateTime.now;
2499
+ const interrupts = yield* Ref.make<
2500
+ ReadonlyArray<ProviderAdapter.ProviderAdapterV2InterruptInput>
2501
+ >([]);
2502
+ const ids = backgroundScenarioIds(`ingestion:${failureMode}`);
2503
+ const { observed, finalEvents } = yield* captureRootRunTermination({
2504
+ key: `ingestion:${failureMode}`,
2505
+ initialProviderTurnId: null,
2506
+ rejectTerminalWrite: failureMode === "ownership-lost",
2507
+ shouldFinalizeRun: () => Effect.succeed(true),
2508
+ startTurn: () =>
2509
+ Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(stopped))),
2510
+ events: () =>
2511
+ Stream.unwrap(
2512
+ Deferred.await(started).pipe(
2513
+ Effect.as(
2514
+ Stream.make({
2515
+ type: "provider_turn.updated" as const,
2516
+ driver,
2517
+ providerTurn: {
2518
+ id: ids.rootProviderTurnId,
2519
+ providerThreadId: ids.providerThreadId,
2520
+ nodeId: ids.rootNodeId,
2521
+ runAttemptId: ids.attemptId,
2522
+ nativeTurnRef: null,
2523
+ ordinal: 1,
2524
+ status: "running" as const,
2525
+ startedAt: now,
2526
+ completedAt: null,
2527
+ },
2528
+ }),
2529
+ ),
2530
+ ),
2531
+ ),
2532
+ ingestNormalized: () => {
2533
+ const error = new ProviderEventIngestor.ProviderEventPublishError({
2534
+ providerSessionId: ProviderSessionId.make(`session:${failureMode}`),
2535
+ eventCount: 1,
2536
+ cause: new Error("SQLITE_FULL: database or disk is full"),
2537
+ });
2538
+ return failureMode === "defect" ? Effect.die(error) : Effect.fail(error);
2539
+ },
2540
+ interruptTurn: (input) =>
2541
+ Ref.update(interrupts, (current) => [...current, input]).pipe(
2542
+ Effect.andThen(
2543
+ input.requestRuntimeRestart === true && failureMode !== "interrupt-failure"
2544
+ ? Ref.set(backgroundWorkRunning, false)
2545
+ : Effect.void,
2546
+ ),
2547
+ Effect.andThen(Deferred.succeed(stopped, undefined)),
2548
+ Effect.andThen(
2549
+ failureMode === "interrupt-failure"
2550
+ ? Effect.fail(
2551
+ new ProviderAdapter.ProviderAdapterInterruptError({
2552
+ driver,
2553
+ providerThreadId: input.providerThread.id,
2554
+ providerTurnId: input.providerTurnId,
2555
+ cause: "provider already exited",
2556
+ }),
2557
+ )
2558
+ : Effect.void,
2559
+ ),
2560
+ ),
2561
+ beforeTerminalWrite: () =>
2562
+ Effect.gen(function* () {
2563
+ const calls = yield* Ref.get(interrupts);
2564
+ assert.equal(calls.length, 1);
2565
+ assert.equal(calls[0]?.providerTurnId, ids.rootProviderTurnId);
2566
+ assert.equal(calls[0]?.providerThread.id, ids.providerThreadId);
2567
+ yield* Ref.set(backgroundWorkAtFinalization, yield* Ref.get(backgroundWorkRunning));
2568
+ return true;
2569
+ }),
2570
+ });
2571
+ if (failureMode === "ownership-lost") {
2572
+ assert.deepEqual(finalEvents, []);
2573
+ assert.deepEqual(observed, []);
2574
+ } else {
2575
+ assert.equal(
2576
+ yield* Ref.get(backgroundWorkAtFinalization),
2577
+ failureMode === "interrupt-failure",
2578
+ );
2579
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2580
+ const repaired = finalEvents.find((event) => event.type === "provider-turn.updated");
2581
+ if (failureMode === "interrupt-failure") {
2582
+ assert.isUndefined(repaired);
2583
+ } else {
2584
+ assert.equal(repaired?.payload.id, ids.rootProviderTurnId);
2585
+ assert.equal(repaired?.payload.status, "interrupted");
2586
+ assert.isNotNull(repaired?.payload.completedAt);
2587
+ }
2588
+ }
2589
+ }),
2590
+);
2591
+
2592
+it.effect("does not interrupt an ended provider when its terminal write fails", () =>
2593
+ Effect.gen(function* () {
2594
+ const interrupts = yield* Ref.make(0);
2595
+ const writes = yield* Ref.make(0);
2596
+ const { observed } = yield* captureRootRunTermination({
2597
+ key: "terminal-write-failure",
2598
+ shouldFinalizeRun: () => Effect.succeed(true),
2599
+ events: (ids) => Stream.make(rootTerminalEvent(ids, "completed")),
2600
+ interruptTurn: () => Ref.update(interrupts, (count) => count + 1),
2601
+ beforeTerminalWrite: () =>
2602
+ Effect.gen(function* () {
2603
+ if ((yield* Ref.updateAndGet(writes, (count) => count + 1)) === 1) {
2604
+ return yield* new EventSink.EventSinkWriteError({ eventCount: 1 });
2605
+ }
2606
+ return true;
2607
+ }),
2608
+ });
2609
+ assert.equal(yield* Ref.get(interrupts), 0);
2610
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2611
+ }),
2612
+);
2613
+
2614
+it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () =>
2615
+ Effect.gen(function* () {
2616
+ const interrupts = yield* Ref.make(0);
2617
+ const { observed } = yield* captureRootRunTermination({
2618
+ key: "unknown-turn-stream-error",
2619
+ initialProviderTurnId: null,
2620
+ shouldFinalizeRun: () => Effect.succeed(true),
2621
+ events: () =>
2622
+ Stream.fail(
2623
+ new ProviderAdapter.ProviderAdapterEventStreamError({
2624
+ driver,
2625
+ providerSessionId: ProviderSessionId.make("session:unknown-turn"),
2626
+ cause: "provider process exited",
2627
+ }),
2628
+ ),
2629
+ interruptTurn: () => Ref.update(interrupts, (count) => count + 1),
2630
+ });
2631
+ assert.equal(yield* Ref.get(interrupts), 0);
2632
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2633
+ }),
2634
+);
2635
+
2636
it.effect("fails the run when its event stream ends without a root terminal", () =>
2637
Effect.gen(function* () {
2638
const { observed, finalEvents } = yield* captureRootRunTermination({
@@ -2564,6 +2710,8 @@ function captureRootRunTermination(input: {
2710
ProviderAdapter.ProviderAdapterV2Error
2711
>;
2712
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
2713
+ readonly interruptTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["interruptTurn"];
2714
+ readonly initialProviderTurnId?: ProviderTurnId | null;
2715
readonly onStartFailure?: (
2716
error: RunExecutionService.RunExecutionServiceV2Error,
2717
) => Effect.Effect<void>;
@@ -2704,6 +2852,7 @@ function captureRootRunTermination(input: {
2852
),
2853
}),
2854
startTurn: input.startTurn ?? (() => Effect.void),
2855
+ interruptTurn: input.interruptTurn ?? (() => Effect.void),
2856
} as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime,
2857
run: {
2858
id: ids.runId,
@@ -2724,7 +2873,10 @@ function captureRootRunTermination(input: {
2873
} as OrchestrationV2ProviderThread,
2874
attempt: {
2875
id: ids.attemptId,
2727
- providerTurnId: ids.rootProviderTurnId,
2876
+ providerTurnId:
2877
+ input.initialProviderTurnId === undefined
2878
+ ? ids.rootProviderTurnId
2879
+ : input.initialProviderTurnId,
2880
} as OrchestrationV2RunAttempt,
2881
attemptId: ids.attemptId,
2882
providerTurnOrdinal: 1,
apps/server/src/orchestration-v2/RunExecutionService.ts
+50
@@ -622,6 +622,7 @@ export const layer: Layer.Layer<
622
readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>;
623
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
624
readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
625
+ readonly interruptedProviderTurn?: OrchestrationV2ProviderTurn;
626
readonly terminal: ProviderTerminalEvent;
627
readonly completedAt?: DateTime.Utc;
628
readonly failureItemPersisted: boolean;
@@ -743,6 +744,24 @@ export const layer: Layer.Layer<
744
: [],
745
events: [
746
...cascadedSubagentEvents,
747
+ ...(input.interruptedProviderTurn === undefined
748
+ ? []
749
+ : [
750
+ {
751
+ id: yield* allocateEventId(),
752
+ type: "provider-turn.updated" as const,
753
+ threadId: input.run.threadId,
754
+ runId: input.run.id,
755
+ nodeId: input.rootNode.id,
756
+ providerInstanceId: input.run.providerInstanceId,
757
+ occurredAt: completedAt,
758
+ payload: {
759
+ ...input.interruptedProviderTurn,
760
+ status: "interrupted" as const,
761
+ completedAt,
762
+ },
763
+ },
764
+ ]),
765
...(finalizedAttempt === null
766
? []
767
: [
@@ -962,6 +981,7 @@ export const layer: Layer.Layer<
981
const terminalEvent = yield* Ref.make<ProviderTerminalEvent | null>(null);
982
const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100);
983
const latestProviderThread = yield* Ref.make(input.providerThread);
984
+ const latestRootProviderTurn = yield* Ref.make<OrchestrationV2ProviderTurn | null>(null);
985
const routeIdentity: ProviderEventRouteIdentity = {
986
threadId: input.run.threadId,
987
runId: input.run.id,
@@ -1318,6 +1338,13 @@ export const layer: Layer.Layer<
1338
Stream.tap(({ event, accepted }) =>
1339
Effect.gen(function* () {
1340
if (!accepted) return;
1341
+ if (
1342
+ event.type === "provider_turn.updated" &&
1343
+ event.providerTurn.runAttemptId === input.attempt.id &&
1344
+ event.providerTurn.providerThreadId === input.providerThread.id
1345
+ ) {
1346
+ yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1347
+ }
1348
let storedEventCount = 0;
1349
const deliveredEvent = filterAssistantEvent(
1350
event,
@@ -1436,6 +1463,28 @@ export const layer: Layer.Layer<
1463
});
1464
if (yield* Ref.get(rootRunFinalized)) return;
1465
const providerThread = yield* Ref.get(latestProviderThread);
1466
+ const routing = yield* Ref.get(eventRouting);
1467
+ // Stop background work too; the failed consumer cannot persist terminal events.
1468
+ const interruptedProviderTurn =
1469
+ routing.rootProviderTurnId !== null && !routing.rootTurnEnded
1470
+ ? yield* input.session
1471
+ .interruptTurn({
1472
+ providerThread,
1473
+ providerTurnId: routing.rootProviderTurnId,
1474
+ requestRuntimeRestart: true,
1475
+ })
1476
+ .pipe(
1477
+ Effect.as(yield* Ref.get(latestRootProviderTurn)),
1478
+ Effect.catchCauseIf(
1479
+ (interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1480
+ (interruptCause) =>
1481
+ Effect.logWarning(
1482
+ "orchestration V2 failed to interrupt turn after ingestion failure",
1483
+ { runId: input.run.id, cause: interruptCause },
1484
+ ).pipe(Effect.as(null)),
1485
+ ),
1486
+ )
1487
+ : null;
1488
const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
1489
const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1490
let nextRetryWarningAt = completedAt;
@@ -1450,6 +1499,7 @@ export const layer: Layer.Layer<
1499
expectedStatus: "running",
1500
},
1501
openRunOwnedSubagents: openSubagents,
1502
+ ...(interruptedProviderTurn === null ? {} : { interruptedProviderTurn }),
1503
completedAt,
1504
terminal: makeFailedTerminalEvent(
1505
makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),
apps/web/src/components/chat/TimelineWorkGroupRows.tsx
+2
-2
@@ -330,12 +330,12 @@ function BackgroundWorktreeSetupChip({ snapshot }: { snapshot: WorktreeSetupSnap
330
variant="ghost-muted"
331
size="micro"
332
className="ml-auto min-w-0 shrink-0"
333
- aria-label={`${scriptName} is still running. Show setup progress.`}
333
+ aria-label={`Background setup is still running: ${scriptName}. Show setup progress.`}
334
/>
335
}
336
>
337
<Spinner className="size-3 shrink-0" />
338
- <span className="truncate">{scriptName}</span>
338
+ <span className="truncate">Background setup</span>
339
</PopoverTrigger>
340
<PopoverPopup side="bottom" align="end" width="lg" padding="compact">
341
<WorktreeSetupCard