The best way to control Claude Code, Codex and more from any computer or phone
fix(server): Stop always completes, and an agent that ignores it can be force stopped (#27252)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 29, 2026 at 22:20 UTC
6b1245cceda1d504b6f16b438e16f2a2e08f3388
37 files changed
+1583
-98
apps/mobile/src/features/threads/ThreadDetailScreen.tsx
+76
@@ -46,6 +46,7 @@ import type {
46
ScopedThreadRef,
47
ServerConfig as T3ServerConfig,
48
UsageLimitsReport,
49
+ RunId,
50
} from "@t3tools/contracts";
51
import {
52
appendCodexArtifactTemplateUsePrompt,
@@ -156,8 +157,10 @@ import {
157
import { appAtomRegistry } from "../../state/atom-registry";
158
import { threadStreamHaptics } from "./threadStreamHaptics";
159
import { useAtomCommand } from "../../state/use-atom-command";
160
+import { orchestrationEnvironment } from "../../state/orchestration";
161
import { useDelayedStatus } from "../../lib/useDelayedStatus";
162
import type { PendingUserInputDraftAnswer } from "@t3tools/client-runtime/state/pending-user-input";
163
+import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution";
164
import type {
165
PendingApproval,
166
PendingUserInput,
@@ -339,6 +342,9 @@ const threadVisitTracker = createThreadVisitTracker({
342
343
// Past the composer's 180 ms height animation.
344
const COMPOSER_HEIGHT_SETTLE_MS = 220;
345
+const NO_RUN_STOP_ATOM = Atom.make<RunStopPresentation | null>(null).pipe(
346
+ Atom.withLabel("mobile-thread-run-stop:empty"),
347
+);
348
349
// Anchors, the submitted message, follow state and pending scrolls belong to
350
// one thread, so the body remounts when the thread changes.
@@ -528,6 +534,50 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) {
534
const unsettleThread = useAtomCommand(threadEnvironment.unsettle, { reportFailure: false });
535
const updateMetadata = useAtomCommand(threadEnvironment.updateMetadata, { reportFailure: false });
536
const visitThread = useAtomCommand(threadEnvironment.visit, { reportFailure: false });
537
+ const anyRunStop = useAtomValue(
538
+ props.selectedThreadDetailRef === null
539
+ ? NO_RUN_STOP_ATOM
540
+ : environmentThreadDetails.runStopAtom(props.selectedThreadDetailRef),
541
+ );
542
+ // A background Stop matters only while that work is still pending.
543
+ const runStop =
544
+ anyRunStop?.background && props.selectedThread.pendingBackgroundTasks.length === 0
545
+ ? null
546
+ : anyRunStop;
547
+ const forceStopThread = useAtomCommand(orchestrationEnvironment.forceStopThread, {
548
+ reportFailure: false,
549
+ });
550
+ const canForceStop =
551
+ useAtomValue(orchestrationEnvironment.forceStopThread.permissionAtom(props.environmentId)) &&
552
+ props.serverConfig?.environment.capabilities.threadForceStop === true;
553
+ const [forceStoppingKey, setForceStoppingKey] = useState<string | null>(null);
554
+ // The run that ignored Stop, so a confirmation left open cannot hit a newer run.
555
+ const forceStop = async (runId: RunId | undefined) => {
556
+ const threadKey = selectedThreadKey;
557
+ setForceStoppingKey(threadKey);
558
+ const result = await forceStopThread({
559
+ environmentId: props.environmentId,
560
+ input:
561
+ runId === undefined
562
+ ? { threadId: props.selectedThread.id }
563
+ : { threadId: props.selectedThread.id, runId },
564
+ });
565
+ setForceStoppingKey((current) => (current === threadKey ? null : current));
566
+ if (result._tag === "Failure") {
567
+ if (!isAtomCommandInterrupted(result)) Dialog.alert("Could not force stop the agent");
568
+ return;
569
+ }
570
+ const others = result.value.sharedWithThreadIds.length;
571
+ if (result.value.outcome === "releasing") {
572
+ Dialog.alert("Turn stopped; the agent is still shutting down");
573
+ }
574
+ if (result.value.outcome === "shared") {
575
+ Dialog.alert(
576
+ "Turn stopped; the agent process kept running",
577
+ `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`,
578
+ );
579
+ }
580
+ };
581
const [statusRequest, setStatusRequest] = useState<{ threadKey: string } | null>(null);
582
const statusPending = statusRequest?.threadKey === selectedThreadKey;
583
const [lifecycleNow, setLifecycleNow] = useState(() => new Date().toISOString());
@@ -619,6 +669,32 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) {
669
if (connectionStatus !== null && !props.draftMode) {
670
return connectionStatus;
671
}
672
+ // Before the request check: an agent stuck on an approval may also ignore Stop.
673
+ if (runStop !== null && contentPresentationKind === "ready") {
674
+ const unresponsive = runStop.state === "unresponsive";
675
+ return {
676
+ kind: "stop",
677
+ unresponsive,
678
+ label: unresponsive ? "Not responding to Stop" : "Stopping…",
679
+ action:
680
+ unresponsive && canForceStop
681
+ ? {
682
+ label: forceStoppingKey === selectedThreadKey ? "Force stopping..." : "Force stop",
683
+ accessibilityLabel: "Force stop the agent",
684
+ disabled: forceStoppingKey === selectedThreadKey,
685
+ onPress: () =>
686
+ Dialog.confirm({
687
+ title: "Force stop the agent?",
688
+ message:
689
+ "It has not answered Stop. Force stop ends the turn and kills the agent's processes, unless other working threads share them.",
690
+ confirmText: "Force stop",
691
+ destructive: true,
692
+ onConfirm: () => void forceStop(runStop.runId),
693
+ }),
694
+ }
695
+ : null,
696
+ };
697
+ }
698
if (props.activePendingApproval !== null || props.activePendingUserInput !== null) {
699
return null;
700
}
apps/mobile/src/features/threads/floating-working-control.tsx
+27
-1
@@ -83,7 +83,9 @@ export function FloatingWorkingControl(props: {
83
const [queueWidth, setQueueWidth] = useState(0);
84
const [actionWidth, setActionWidth] = useState(0);
85
const action =
86
- props.status?.kind === "background" || props.status?.kind === "lifecycle"
86
+ props.status?.kind === "background" ||
87
+ props.status?.kind === "lifecycle" ||
88
+ props.status?.kind === "stop"
89
? props.status.action
90
: null;
91
const [agentsWidth, setAgentsWidth] = useState(0);
@@ -471,6 +473,30 @@ function FloatingStatusLabel(props: {
473
</StatusLabelRow>
474
);
475
}
476
+ if (props.status.kind === "stop") {
477
+ return (
478
+ <StatusLabelRow
479
+ key="stop"
480
+ accessibilityLabel={props.status.label}
481
+ className="gap-2"
482
+ onLayout={props.onLayout}
483
+ >
484
+ {props.status.unresponsive ? (
485
+ <SymbolView
486
+ name="exclamationmark.triangle"
487
+ size={13}
488
+ tintColorClassName="accent-warning-foreground"
489
+ type="monochrome"
490
+ />
491
+ ) : (
492
+ <AppActivityIndicator size="small" colorClassName="accent-icon-muted" />
493
+ )}
494
+ <Text className="shrink font-t3-medium text-xs text-foreground" numberOfLines={1}>
495
+ {props.status.label}
496
+ </Text>
497
+ </StatusLabelRow>
498
+ );
499
+ }
500
if (props.status.kind === "goal") {
501
return (
502
<StatusLabelRow
apps/mobile/src/features/threads/floating-working-status.ts
+6
@@ -31,6 +31,12 @@ export type FloatingWorkingStatus =
31
readonly label: string;
32
readonly action: FloatingStatusAction;
33
}
34
+ | {
35
+ readonly kind: "stop";
36
+ readonly label: string;
37
+ readonly unresponsive: boolean;
38
+ readonly action: FloatingStatusAction | null;
39
+ }
40
| { readonly kind: "goal"; readonly label: string; readonly accessibilityLabel: string }
41
| { readonly kind: "preparing"; readonly label: string }
42
| {
apps/server/src/auth/RpcAuthorization.ts
+1
@@ -42,6 +42,7 @@ export const RPC_REQUIRED_SCOPES = {
42
[ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: AuthOrchestrationOperateScope,
43
[ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: AuthOrchestrationReadScope,
44
[ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope,
45
+ [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope,
46
[ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: AuthOrchestrationReadScope,
47
[ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: AuthOrchestrationReadScope,
48
[ORCHESTRATION_V2_WS_METHODS.searchThreads]: AuthOrchestrationReadScope,
apps/server/src/environment/ServerEnvironment.ts
+1
@@ -240,6 +240,7 @@ export const make = Effect.gen(function* () {
240
projectSettingsOverrides: true,
241
projectProviderInstanceOverrides: true,
242
threadSnooze: true,
243
+ threadForceStop: true,
244
environmentThemes: true,
245
usageLimitSources: true,
246
providerWorkspaceSnapshotsOnDemand: true,
apps/server/src/observability/RpcInstrumentation.ts
+1
@@ -14,6 +14,7 @@ const RPC_AGGREGATES = {
14
[ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: "orchestrationV2",
15
[ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: "orchestration",
16
[ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: "orchestration",
17
+ [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: "orchestration",
18
[ORCHESTRATION_V2_WS_METHODS.getTurnItem]: "orchestration",
19
[ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: "orchestration",
20
[ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: "orchestration",
apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts
+90
@@ -7498,6 +7498,96 @@ describe("AcpAdapterV2", () => {
7498
}).pipe(Effect.provide(layerTest), Effect.scoped),
7499
);
7500
7501
+ it.live("lets a caller abandon Stop while hard teardown stalls, and finishes the teardown", () =>
7502
+ Effect.gen(function* () {
7503
+ const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
7504
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
7505
+ const path = yield* Path.Path;
7506
+ const selfInvocation = yield* resolveSelfInvocation();
7507
+ const mockAgentPath = yield* path.fromFileUrl(
7508
+ new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
7509
+ );
7510
+ const protocolEvents = yield* Queue.bounded<EffectAcpProtocol.AcpProtocolLogEvent>(256);
7511
+ const teardownStarted = yield* Deferred.make<void>();
7512
+ const releaseTeardown = yield* Deferred.make<void>();
7513
+ let runtimeOrdinalSeen = 0;
7514
+ const instanceId = ProviderInstanceId.make("acp-test");
7515
+ const adapter = yield* makeAcpAdapterV2({
7516
+ instanceId,
7517
+ flavor: {
7518
+ driver: ACP_TEST_DRIVER,
7519
+ capabilities: AcpProviderCapabilitiesV2,
7520
+ interruptPromptOnCancel: true,
7521
+ restartRuntimeAfterInterrupt: true,
7522
+ terminateRuntimeProcessGroupOnInterrupt: true,
7523
+ makeRuntime: makeMockRuntime({
7524
+ childProcessSpawner,
7525
+ mockAgentPath,
7526
+ ownDetachedProcessGroup: true,
7527
+ processGroupPlatform: "win32",
7528
+ windowsProcessTreeTerminator: () =>
7529
+ Deferred.succeed(teardownStarted, undefined).pipe(
7530
+ Effect.andThen(Deferred.await(releaseTeardown)),
7531
+ ),
7532
+ environment: (runtimeOrdinal) => {
7533
+ runtimeOrdinalSeen = runtimeOrdinal;
7534
+ return runtimeOrdinal === 1 ? { T3_ACP_HANG_PROMPT_FOREVER: "1" } : {};
7535
+ },
7536
+ protocolEvents,
7537
+ }),
7538
+ },
7539
+ selfInvocation,
7540
+ });
7541
+ const threadId = ThreadId.make("thread-acp-abandoned-hard-teardown");
7542
+ const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({
7543
+ runtimeMode: "full-access",
7544
+ interactionMode: "default",
7545
+ cwd: process.cwd(),
7546
+ });
7547
+ const modelSelection = { instanceId, model: "default" } as const;
7548
+ const runtime = yield* adapter.openSession({
7549
+ threadId,
7550
+ providerSessionId: ProviderSessionId.make("provider-session-acp-abandoned-hard-teardown"),
7551
+ modelSelection,
7552
+ runtimePolicy,
7553
+ });
7554
+ const providerThread = yield* runtime.ensureThread({
7555
+ threadId,
7556
+ modelSelection,
7557
+ runtimePolicy,
7558
+ });
7559
+ const now = yield* DateTime.now;
7560
+ yield* runtime
7561
+ .startTurn(
7562
+ makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }),
7563
+ )
7564
+ .pipe(Effect.forkScoped);
7565
+ const providerTurnId = idAllocator.derive.providerTurn({
7566
+ driver: ACP_TEST_DRIVER,
7567
+ nativeTurnId: acpScopedNativeId(instanceId, "mock-session-1:turn:1"),
7568
+ });
7569
+ yield* pollProtocolMethods(protocolEvents);
7570
+ const interruptFiber = yield* runtime
7571
+ .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true })
7572
+ .pipe(Effect.forkScoped);
7573
+ yield* Deferred.await(teardownStarted);
7574
+
7575
+ // The session manager's Stop timeout interrupts this call; it must not wait for teardown.
7576
+ yield* Fiber.interrupt(interruptFiber);
7577
+
7578
+ const startFiber = yield* runtime
7579
+ .startTurn(
7580
+ makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }),
7581
+ )
7582
+ .pipe(Effect.forkScoped);
7583
+ yield* Deferred.succeed(releaseTeardown, undefined);
7584
+ yield* Fiber.join(startFiber);
7585
+ assert.equal(runtimeOrdinalSeen, 2);
7586
+ const replacementMethods = yield* pollProtocolMethods(protocolEvents);
7587
+ assert.equal(replacementMethods.filter((method) => method === "session/resume").length, 1);
7588
+ }).pipe(Effect.provide(layerTest), Effect.scoped),
7589
+ );
7590
+
7591
it.effect(
7592
"direct Stop on a deferred subagent hold terminalizes the subagent and does not carry it forward",
7593
() =>
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+20
-26
@@ -322,6 +322,8 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500;
322
// A steer waits at most this long, from its request, for its foreground
323
// subagents to move to the background before it is sent.
324
const BACKGROUND_MOVE_TIMEOUT_MS = 5_000;
325
+// How long a Stop waits for the CLI to exit before the user is offered Force stop.
326
+const CLAUDE_STOP_CONFIRM_TIMEOUT_MS = 10_000;
327
328
const CLAUDE_GROUP_WATCH_SCRIPT = [
329
"{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } </dev/null >/dev/null 2>&1 &",
@@ -10471,44 +10473,36 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10473
yield* existing.query.interrupt;
10474
yield* existing.query.close.pipe(Effect.ignore);
10475
const closed = yield* Deferred.await(existing.closed).pipe(
10474
- Effect.timeoutOption("10 seconds"),
10476
+ Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS),
10477
);
10478
if (Option.isSome(closed)) {
10479
return;
10480
}
10479
-
10480
- const completedAt = yield* DateTime.now;
10481
+ // The turn stays open: the CLI may still be working, and the user decides
10482
+ // whether to Force stop it.
10483
yield* Effect.logWarning("orchestration-v2.claude-query-interrupt-timeout", {
10484
providerSessionId: input.providerSessionId,
10485
providerThreadId: turnInput.providerThread.id,
10486
providerTurnId: turnInput.providerTurnId,
10487
});
10486
- const ownedLiveQuery = yield* Ref.modify(queryContext, (current) =>
10487
- current?.query === existing.query ? [true, null] : [false, current],
10488
- );
10489
- if (ownedLiveQuery) {
10490
- existing.stopping = true;
10491
- yield* finalizeActiveTurnAfterQueryExit(existing);
10492
- yield* endSubagentCallsOfClosedQuery(existing.nativeThreadId, "interrupted");
10493
- } else {
10494
- yield* finalizeActiveTurn({
10495
- context: currentTurn,
10496
- status: "interrupted",
10497
- completedAt,
10498
- });
10499
- }
10500
- yield* Deferred.succeed(existing.closed, undefined);
10488
+ return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
10489
+ driver: CLAUDE_PROVIDER,
10490
+ providerThreadId: turnInput.providerThread.id,
10491
+ providerTurnId: turnInput.providerTurnId,
10492
+ timeoutMs: CLAUDE_STOP_CONFIRM_TIMEOUT_MS,
10493
+ });
10494
},
10495
(effect, turnInput) =>
10496
effect.pipe(
10504
- Effect.mapError(
10505
- (cause) =>
10506
- new ProviderAdapter.ProviderAdapterInterruptError({
10507
- driver: CLAUDE_PROVIDER,
10508
- providerThreadId: turnInput.providerThread.id,
10509
- providerTurnId: turnInput.providerTurnId,
10510
- cause,
10511
- }),
10497
+ Effect.mapError((cause) =>
10498
+ cause._tag === "ProviderAdapterInterruptTimeoutError"
10499
+ ? cause
10500
+ : new ProviderAdapter.ProviderAdapterInterruptError({
10501
+ driver: CLAUDE_PROVIDER,
10502
+ providerThreadId: turnInput.providerThread.id,
10503
+ providerTurnId: turnInput.providerTurnId,
10504
+ cause,
10505
+ }),
10506
),
10507
),
10508
);
apps/server/src/orchestration-v2/EffectWorker.test.ts
+17
-3
@@ -111,6 +111,8 @@ function layerExecutorFor(input: {
111
closeInstance: () => Effect.void,
112
release: () => record("release"),
113
detach: () => record("detach"),
114
+ otherWorkingThreads: () => Effect.succeed([]),
115
+ releaseUnlessBusy: () => Effect.succeed(true),
116
}),
117
),
118
Layer.succeed(
@@ -167,7 +169,18 @@ function layerExecutorFor(input: {
169
);
170
}
171
170
-it.effect("settles a stopped run when its adapter has already lost the native turn", () =>
172
+it.effect.each([
173
+ {
174
+ name: "settles a stopped run when its adapter has already lost the native turn",
175
+ unresponsive: false,
176
+ dispatched: ["thread.background-work.settle"],
177
+ },
178
+ {
179
+ name: "leaves the run running and flags the Stop when the provider never confirms it",
180
+ unresponsive: true,
181
+ dispatched: ["thread.interrupt.unresponsive"],
182
+ },
183
+])("$name", ({ unresponsive, dispatched }) =>
184
Effect.gen(function* () {
185
const now = yield* DateTime.now;
186
const events = yield* Ref.make<ReadonlyArray<string>>([]);
@@ -178,7 +191,8 @@ it.effect("settles a stopped run when its adapter has already lost the native tu
191
threadId,
192
operation: "interrupt",
193
providerTurnId,
181
- cause: "Provider turn is not active.",
194
+ ...(unresponsive ? { unresponsive } : {}),
195
+ cause: unresponsive ? "Timed out." : "Provider turn is not active.",
196
}),
197
threads: {
198
dispatch: (command) =>
@@ -199,7 +213,7 @@ it.effect("settles a stopped run when its adapter has already lost the native tu
213
},
214
});
215
}).pipe(Effect.provide(layer));
202
- assert.deepEqual(yield* Ref.get(events), ["thread.background-work.settle"]);
216
+ assert.deepEqual(yield* Ref.get(events), dispatched);
217
}),
218
);
219
apps/server/src/orchestration-v2/EffectWorker.ts
+32
-17
@@ -177,6 +177,13 @@ export const layerExecutor: Layer.Layer<
177
cause,
178
}),
179
),
180
+ Effect.map((outcome) => ({ unresponsive: false, outcome }) as const),
181
+ // Settling here would end the run while the agent may still be running; the
182
+ // user decides with Force stop instead.
183
+ Effect.catchIf(
184
+ (cause) => request.subagent === undefined && cause.unresponsive === true,
185
+ () => Effect.succeed({ unresponsive: true, outcome: undefined } as const),
186
+ ),
187
Effect.catchIf(
188
(cause) =>
189
request.subagent === undefined &&
@@ -184,27 +191,35 @@ export const layerExecutor: Layer.Layer<
191
request.type,
192
Cause.pretty(Cause.fail(cause)),
193
),
187
- () => Effect.void,
194
+ () => Effect.succeed({ unresponsive: false, outcome: undefined } as const),
195
),
189
- Effect.andThen((outcome) =>
190
- request.subagent !== undefined && outcome === undefined
191
- ? Effect.void
192
- : threads.dispatch({
193
- type: "thread.background-work.settle",
194
- commandId: CommandId.make(`${effect.id}:background-work-settled`),
195
- ...(request.subagent === undefined || outcome === undefined
196
- ? {}
197
- : {
198
- subagent: {
199
- id: request.subagent.id,
200
- stoppedAt: DateTime.makeUnsafe(effect.createdAt),
201
- outcome,
202
- },
203
- }),
196
+ Effect.andThen(({ unresponsive, outcome }) =>
197
+ unresponsive
198
+ ? threads.dispatch({
199
+ type: "thread.interrupt.unresponsive",
200
+ commandId: CommandId.make(`${effect.id}:unresponsive`),
201
threadId: effect.threadId,
202
providerThreadId: request.providerThreadId,
203
providerTurnId: request.providerTurnId,
207
- }),
204
+ })
205
+ : request.subagent !== undefined && outcome === undefined
206
+ ? Effect.void
207
+ : threads.dispatch({
208
+ type: "thread.background-work.settle",
209
+ commandId: CommandId.make(`${effect.id}:background-work-settled`),
210
+ ...(request.subagent === undefined || outcome === undefined
211
+ ? {}
212
+ : {
213
+ subagent: {
214
+ id: request.subagent.id,
215
+ stoppedAt: DateTime.makeUnsafe(effect.createdAt),
216
+ outcome,
217
+ },
218
+ }),
219
+ threadId: effect.threadId,
220
+ providerThreadId: request.providerThreadId,
221
+ providerTurnId: request.providerTurnId,
222
+ }),
223
),
224
Effect.mapError(
225
(cause) =>
apps/server/src/orchestration-v2/Orchestrator.ts
+58
-1
@@ -536,6 +536,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId {
536
case "checkpoint.rollback":
537
case "checkpoint.rollback.fail":
538
case "thread.background-work.settle":
539
+ case "thread.interrupt.unresponsive":
540
case "thread.stop":
541
case "provider.switch":
542
return command.threadId;
@@ -10278,7 +10279,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10279
run,
10280
rootNode,
10281
providerThread,
10281
- reason: yield* runInterruptReason(input.command, input.events, run.id),
10282
+ reason:
10283
+ (input.command.type === "thread.background-work.settle"
10284
+ ? input.command.reason
10285
+ : undefined) ?? (yield* runInterruptReason(input.command, input.events, run.id)),
10286
completedAt: input.now,
10287
}),
10288
});
@@ -10525,6 +10529,55 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10529
yield* closePendingLinkRequests({ ...input, threadId: thread.id });
10530
});
10531
10532
+ const dispatchInterruptUnresponsive = Effect.fn("Orchestrator.dispatchInterruptUnresponsive")(
10533
+ function* (
10534
+ command: Extract<
10535
+ OrchestrationV2InternalCommand,
10536
+ { readonly type: "thread.interrupt.unresponsive" }
10537
+ >,
10538
+ events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
10539
+ ) {
10540
+ const { providerTurn } = yield* projectionStore
10541
+ .getProviderControlContext(command.threadId, {
10542
+ providerThreadId: command.providerThreadId,
10543
+ providerTurnId: command.providerTurnId,
10544
+ })
10545
+ .pipe(mapDispatchError(command));
10546
+ const { runs } = yield* projectionStore
10547
+ .getThreadRecords(command.threadId, ["runs"])
10548
+ .pipe(mapDispatchError(command));
10549
+ const run = runs.find(
10550
+ (candidate) =>
10551
+ providerTurn !== undefined && candidate.activeAttemptId === providerTurn.runAttemptId,
10552
+ );
10553
+ // A settled run still counts: Stop may have targeted its background work.
10554
+ if (run === undefined) return;
10555
+ const { turnItems } = yield* projectionStore
10556
+ .getThreadRecords(command.threadId, ["turnItems"], {
10557
+ turnItemTypes: ["run_interrupt_request", "run_interrupt_result"],
10558
+ turnItemRunIds: [run.id],
10559
+ })
10560
+ .pipe(mapDispatchError(command));
10561
+ // A provider that stopped late already wrote the result; it needs no Force stop.
10562
+ if (turnItems.some((item) => item.type === "run_interrupt_result")) return;
10563
+ const request = turnItems.find((item) => item.type === "run_interrupt_request");
10564
+ if (request?.type !== "run_interrupt_request" || request.unresponsiveAt !== undefined) return;
10565
+ const now = yield* DateTime.now;
10566
+ yield* emit(
10567
+ events,
10568
+ command,
10569
+ )({
10570
+ type: "turn-item.updated",
10571
+ threadId: command.threadId,
10572
+ runId: run.id,
10573
+ ...(request.nodeId === null ? {} : { nodeId: request.nodeId }),
10574
+ providerInstanceId: run.providerInstanceId,
10575
+ occurredAt: now,
10576
+ payload: { ...request, unresponsiveAt: now, updatedAt: now },
10577
+ });
10578
+ },
10579
+ );
10580
+
10581
const dispatchBackgroundWorkSettle = Effect.fn("Orchestrator.dispatchBackgroundWorkSettle")(
10582
function* (
10583
command: Extract<
@@ -12902,6 +12955,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12955
case "thread.background-work.settle":
12956
yield* dispatchBackgroundWorkSettle(command, events, effects);
12957
break;
12958
+ case "thread.interrupt.unresponsive":
12959
+ yield* dispatchInterruptUnresponsive(command, events);
12960
+ break;
12961
case "thread.fork":
12962
yield* dispatchThreadFork(command, events);
12963
break;
@@ -13032,6 +13088,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13088
(planned) =>
13089
planned.events.length > 0 ||
13090
command.type === "thread.background-work.settle" ||
13091
+ command.type === "thread.interrupt.unresponsive" ||
13092
command.type === "thread.stop" ||
13093
command.type === "subagent.stop" ||
13094
// Closing a share again, or after it ended, is not an error.
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+157
-1
@@ -338,6 +338,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
338
readonly backgroundTasksEndWithProcess?: boolean;
339
readonly hangSessionScopeClose?: boolean;
340
readonly startTurn?: Effect.Effect<void>;
341
+ readonly interruptTurn?: Effect.Effect<void>;
342
readonly beforeUnload?: Effect.Effect<void>;
343
readonly spawnBeforeOpen?: boolean;
344
readonly scopeCloseReached?: Deferred.Deferred<void>;
@@ -437,7 +438,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
438
Ref.update(state, (current) => ({
439
...current,
440
interruptCount: current.interruptCount + 1,
440
- })),
441
+ })).pipe(Effect.andThen(options.interruptTurn ?? Effect.void)),
442
unloadThread: ({ providerThread }) =>
443
(options.beforeUnload ?? Effect.void).pipe(
444
Effect.andThen(
@@ -485,6 +486,7 @@ function layerTest(input: {
486
readonly backgroundTasksEndWithProcess?: boolean;
487
readonly hangSessionScopeClose?: boolean;
488
readonly startTurn?: Effect.Effect<void>;
489
+ readonly interruptTurn?: Effect.Effect<void>;
490
readonly beforeUnload?: Effect.Effect<void>;
491
readonly spawnBeforeOpen?: boolean;
492
readonly scopeCloseReached?: Deferred.Deferred<void>;
@@ -520,6 +522,7 @@ function layerTest(input: {
522
? {}
523
: { hangSessionScopeClose: input.hangSessionScopeClose }),
524
...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }),
525
+ ...(input.interruptTurn === undefined ? {} : { interruptTurn: input.interruptTurn }),
526
...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }),
527
...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }),
528
...(input.scopeCloseReached === undefined
@@ -1367,6 +1370,159 @@ it.effect(
1370
}),
1371
);
1372
1373
+it.effect("ProviderSessionManagerV2 gives up on a provider that never confirms a Stop", () =>
1374
+ Effect.gen(function* () {
1375
+ const state = yield* Ref.make(emptyState);
1376
+ yield* Effect.gen(function* () {
1377
+ const eventSink = yield* EventSink.EventSinkV2;
1378
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
1379
+ const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1380
+ const now = yield* DateTime.now;
1381
+ const threadId = ThreadId.make("thread-provider-session-manager-unresponsive-stop");
1382
+ const providerSessionId = yield* idAllocator.allocate.providerSession({
1383
+ providerInstanceId: modelSelection.instanceId,
1384
+ threadId,
1385
+ });
1386
+ yield* eventSink.write({
1387
+ events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
1388
+ });
1389
+ const runtime = yield* manager.open({
1390
+ threadId,
1391
+ providerSessionId,
1392
+ modelSelection,
1393
+ runtimePolicy,
1394
+ });
1395
+ const stopping = yield* runtime
1396
+ .interruptTurn({
1397
+ providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }),
1398
+ providerTurnId: idAllocator.derive.providerTurn({
1399
+ driver: CODEX_DRIVER,
1400
+ nativeTurnId: "native-turn-unresponsive",
1401
+ }),
1402
+ })
1403
+ .pipe(Effect.flip, Effect.forkChild);
1404
+ yield* TestClock.adjust(ProviderSessionManager.PROVIDER_INTERRUPT_TIMEOUT_MS);
1405
+ assert.equal((yield* Fiber.join(stopping))._tag, "ProviderAdapterInterruptTimeoutError");
1406
+ }).pipe(
1407
+ Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, interruptTurn: Effect.never })),
1408
+ );
1409
+ }),
1410
+);
1411
+
1412
+it.effect(
1413
+ "ProviderSessionManagerV2 keeps a shared session a sibling is busy on when releasing",
1414
+ () =>
1415
+ Effect.gen(function* () {
1416
+ const state = yield* Ref.make(emptyState);
1417
+ yield* Effect.gen(function* () {
1418
+ const eventSink = yield* EventSink.EventSinkV2;
1419
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
1420
+ const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1421
+ const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1422
+ const now = yield* DateTime.now;
1423
+ const stopping = ThreadId.make("thread-provider-session-manager-release-stopping");
1424
+ const sibling = ThreadId.make("thread-provider-session-manager-release-sibling");
1425
+ const providerSessionId = idAllocator.derive.providerSession({
1426
+ providerInstanceId: modelSelection.instanceId,
1427
+ });
1428
+ yield* eventSink.write({
1429
+ events: [
1430
+ yield* makeThreadCreatedEvent({ idAllocator, threadId: stopping, now }),
1431
+ yield* makeThreadCreatedEvent({ idAllocator, threadId: sibling, now }),
1432
+ ],
1433
+ });
1434
+ const runtime = yield* manager.open({
1435
+ threadId: stopping,
1436
+ providerSessionId,
1437
+ modelSelection,
1438
+ runtimePolicy,
1439
+ });
1440
+ yield* manager.open({
1441
+ threadId: sibling,
1442
+ providerSessionId,
1443
+ modelSelection,
1444
+ runtimePolicy,
1445
+ });
1446
+ const providerThreadOf = (threadId: ThreadId) =>
1447
+ makeProviderThread({
1448
+ idAllocator,
1449
+ threadId,
1450
+ providerSessionId,
1451
+ now,
1452
+ nativeThreadId: `native-${threadId}`,
1453
+ });
1454
+ const runId = idAllocator.derive.run({ threadId: sibling, ordinal: 1 });
1455
+ yield* runtime.startTurn({
1456
+ appThread: (yield* projectionStore.getThreadProjection(sibling)).thread,
1457
+ threadId: sibling,
1458
+ runId,
1459
+ runOrdinal: 1,
1460
+ providerTurnOrdinal: 1,
1461
+ attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }),
1462
+ rootNodeId: idAllocator.derive.rootNode({ runId }),
1463
+ providerThread: providerThreadOf(sibling),
1464
+ message: {
1465
+ createdBy: "user",
1466
+ creationSource: "web",
1467
+ messageId: yield* idAllocator.allocate.message({ threadId: sibling, ordinal: 1 }),
1468
+ text: "turn",
1469
+ attachments: [],
1470
+ },
1471
+ modelSelection,
1472
+ runtimePolicy,
1473
+ });
1474
+ const release = (own: ThreadId) =>
1475
+ manager.releaseUnlessBusy({
1476
+ providerSessionId,
1477
+ ownProviderThreadIds: [providerThreadOf(own).id],
1478
+ reason: "manual_shutdown",
1479
+ });
1480
+
1481
+ assert.isFalse(yield* release(stopping));
1482
+ assert.isTrue(yield* manager.isLive(providerSessionId));
1483
+ assert.isTrue(yield* release(sibling));
1484
+ assert.isFalse(yield* manager.isLive(providerSessionId));
1485
+
1486
+ // A turn that starts on a runtime released meanwhile fails instead of running on it.
1487
+ const startOnReleasedRuntime = Effect.gen(function* () {
1488
+ const lateRunId = idAllocator.derive.run({ threadId: stopping, ordinal: 1 });
1489
+ return yield* Effect.flip(
1490
+ runtime.startTurn({
1491
+ appThread: (yield* projectionStore.getThreadProjection(stopping)).thread,
1492
+ threadId: stopping,
1493
+ runId: lateRunId,
1494
+ runOrdinal: 1,
1495
+ providerTurnOrdinal: 1,
1496
+ attemptId: idAllocator.derive.runAttempt({ runId: lateRunId, attemptOrdinal: 1 }),
1497
+ rootNodeId: idAllocator.derive.rootNode({ runId: lateRunId }),
1498
+ providerThread: providerThreadOf(stopping),
1499
+ message: {
1500
+ createdBy: "user",
1501
+ creationSource: "web",
1502
+ messageId: yield* idAllocator.allocate.message({ threadId: stopping, ordinal: 1 }),
1503
+ text: "turn",
1504
+ attachments: [],
1505
+ },
1506
+ modelSelection,
1507
+ runtimePolicy,
1508
+ }),
1509
+ );
1510
+ });
1511
+ assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError");
1512
+
1513
+ // Nor may it leave its turn busy on a replacement opened under the same id.
1514
+ yield* manager.open({
1515
+ threadId: sibling,
1516
+ providerSessionId,
1517
+ modelSelection,
1518
+ runtimePolicy,
1519
+ });
1520
+ assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError");
1521
+ assert.isTrue(yield* release(sibling));
1522
+ }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60 * 60_000 })));
1523
+ }),
1524
+);
1525
+
1526
it.effect("ProviderSessionManagerV2 stops a session still opening when its layer shuts down", () =>
1527
Effect.gen(function* () {
1528
const state = yield* Ref.make(emptyState);
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+128
-11
@@ -57,11 +57,19 @@ import * as ProjectionStore from "./ProjectionStore.ts";
57
const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
58
const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000;
59
const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000;
60
+/** Past adapters' own 10 s stop fallbacks, so they settle first when the provider answers at all. */
61
+export const PROVIDER_INTERRUPT_TIMEOUT_MS = 15 * 1000;
62
63
const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThreadId}#`;
64
const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) =>
65
`${busyTurnPrefix(providerThreadId)}${runOrdinal}`;
66
const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000;
67
+const WORKING_RUN_STATUSES: ReadonlySet<string> = new Set([
68
+ "preparing",
69
+ "starting",
70
+ "running",
71
+ "waiting",
72
+]);
73
74
export const ProviderSessionReleaseReason = Schema.Literals([
75
"idle_timeout",
@@ -181,6 +189,25 @@ export interface ProviderSessionManagerV2Shape {
189
readonly detail?: string;
190
readonly revokeMcpCredential?: boolean;
191
}) => Effect.Effect<void, ProviderSessionManagerV2Error>;
192
+ /**
193
+ * Releases the session unless, at the moment of removal, a turn of a provider thread outside
194
+ * `ownProviderThreadIds` is busy on it. Checked atomically with turn starts. Succeeds with
195
+ * whether it was released.
196
+ */
197
+ readonly releaseUnlessBusy: (input: {
198
+ readonly providerSessionId: ProviderSessionId;
199
+ readonly ownProviderThreadIds: ReadonlyArray<ProviderThreadId>;
200
+ readonly reason: ProviderSessionReleaseReason;
201
+ readonly detail?: string;
202
+ }) => Effect.Effect<boolean, ProviderSessionManagerV2Error>;
203
+ /**
204
+ * Other threads with a run or background work on this session's shared process; empty when
205
+ * it serves one thread.
206
+ */
207
+ readonly otherWorkingThreads: (input: {
208
+ readonly providerSessionId: ProviderSessionId;
209
+ readonly threadId: ThreadId;
210
+ }) => Effect.Effect<ReadonlyArray<ThreadId>>;
211
}
212
213
export class ProviderSessionManagerV2 extends Context.Service<
@@ -895,6 +922,7 @@ export const layerWithOptions = (
922
function* (input: {
923
readonly providerSessionId: ProviderSessionId;
924
readonly onlyIfIdleGeneration?: number;
925
+ readonly keepIfBusyOtherThan?: ReadonlyArray<ProviderThreadId>;
926
}): Effect.fn.Return<readonly [Option.Option<LiveSessionEntry>, DateTime.Utc]> {
927
const key = sessionKey(input.providerSessionId);
928
const candidate = (yield* Ref.get(sessions)).get(key);
@@ -914,6 +942,15 @@ export const layerWithOptions = (
942
) {
943
return ["kept", current] as const;
944
}
945
+ const own = input.keepIfBusyOtherThan;
946
+ if (
947
+ own !== undefined &&
948
+ [...existing.busyTurns].some(
949
+ (turnKey) => !own.some((id) => turnKey.startsWith(busyTurnPrefix(id))),
950
+ )
951
+ ) {
952
+ return ["kept", current] as const;
953
+ }
954
const updated = new Map(current);
955
updated.delete(key);
956
return ["removed", updated] as const;
@@ -972,6 +1009,7 @@ export const layerWithOptions = (
1009
readonly detachedThreadId?: ThreadId;
1010
readonly cancelIdleFiber?: boolean;
1011
readonly onlyIfIdleGeneration?: number;
1012
+ readonly keepIfBusyOtherThan?: ReadonlyArray<ProviderThreadId>;
1013
readonly gracefulSubscribers?: boolean;
1014
}) =>
1015
Effect.acquireUseRelease(
@@ -1380,15 +1418,21 @@ export const layerWithOptions = (
1418
.pipe(Effect.ensuring(Effect.sync(dropReservation)));
1419
});
1420
1383
- const markBusy = (providerSessionId: ProviderSessionId, turnKey: string) =>
1421
+ // Marks only the entry this runtime belongs to: a released or replaced session must not
1422
+ // gain a busy turn that never ends. Succeeds with whether the turn was marked.
1423
+ const markBusy = (
1424
+ providerSessionId: ProviderSessionId,
1425
+ turnKey: string,
1426
+ runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime,
1427
+ ) =>
1428
withActivityError(
1429
providerSessionId,
1430
Effect.gen(function* () {
1431
const key = sessionKey(providerSessionId);
1388
- const idleFiber = yield* Ref.modify(sessions, (current) => {
1432
+ const marked = yield* Ref.modify(sessions, (current) => {
1433
const entry = current.get(key);
1390
- if (entry === undefined) {
1391
- return [null, current] as const;
1434
+ if (entry === undefined || entry.runtime !== runtime) {
1435
+ return [undefined, current] as const;
1436
}
1437
const updated = new Map(current);
1438
updated.set(key, {
@@ -1397,9 +1441,11 @@ export const layerWithOptions = (
1441
idleFiber: null,
1442
pinnedSinceMs: null,
1443
});
1400
- return [entry.idleFiber, updated] as const;
1444
+ return [{ idleFiber: entry.idleFiber }, updated] as const;
1445
});
1402
- yield* cancelIdleFiber(idleFiber);
1446
+ if (marked === undefined) return false;
1447
+ yield* cancelIdleFiber(marked.idleFiber);
1448
+ return true;
1449
}),
1450
);
1451
@@ -1785,11 +1831,31 @@ export const layerWithOptions = (
1831
),
1832
Effect.andThen(
1833
Effect.acquireUseRelease(
1788
- observeActivity(
1834
+ markBusy(
1835
providerSessionId,
1790
- markBusy(
1791
- providerSessionId,
1792
- busyTurnKey(input.providerThread.id, input.runOrdinal),
1836
+ busyTurnKey(input.providerThread.id, input.runOrdinal),
1837
+ runtime,
1838
+ ).pipe(
1839
+ Effect.catchCause((cause) =>
1840
+ Effect.logWarning("orchestration-v2.driver-session.activity-failed", {
1841
+ providerSessionId,
1842
+ cause,
1843
+ }).pipe(Effect.as(true)),
1844
+ ),
1845
+ // A release between attach and markBusy leaves nothing to mark: fail the
1846
+ // start instead of running it on a runtime being torn down.
1847
+ Effect.flatMap((marked) =>
1848
+ marked
1849
+ ? Effect.void
1850
+ : Effect.fail(
1851
+ new ProviderAdapter.ProviderAdapterTurnStartError({
1852
+ driver: runtime.driver,
1853
+ threadId: input.threadId,
1854
+ providerThreadId: input.providerThread.id,
1855
+ runId: input.runId,
1856
+ cause: "The provider session was released while the turn started.",
1857
+ }),
1858
+ ),
1859
),
1860
),
1861
() =>
@@ -1808,9 +1874,26 @@ export const layerWithOptions = (
1874
observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
1875
Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))),
1876
),
1877
+ // A provider starved of memory can stop answering, so no caller may wait on it unbounded.
1878
interruptTurn: (input) =>
1879
observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
1813
- Effect.andThen(runtime.interruptTurn(input).pipe(turnMetrics("interrupt"))),
1880
+ Effect.andThen(
1881
+ runtime.interruptTurn(input).pipe(
1882
+ Effect.timeoutOrElse({
1883
+ duration: PROVIDER_INTERRUPT_TIMEOUT_MS,
1884
+ orElse: () =>
1885
+ Effect.fail(
1886
+ new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
1887
+ driver: runtime.driver,
1888
+ providerThreadId: input.providerThread.id,
1889
+ providerTurnId: input.providerTurnId,
1890
+ timeoutMs: PROVIDER_INTERRUPT_TIMEOUT_MS,
1891
+ }),
1892
+ ),
1893
+ }),
1894
+ turnMetrics("interrupt"),
1895
+ ),
1896
+ ),
1897
),
1898
respondToRuntimeRequest: (input) =>
1899
observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
@@ -2276,6 +2359,40 @@ export const layerWithOptions = (
2359
),
2360
),
2361
release: releaseEntry,
2362
+ releaseUnlessBusy: Effect.fn("ProviderSessionManager.releaseUnlessBusy")(function* (input) {
2363
+ const key = sessionKey(input.providerSessionId);
2364
+ const before = (yield* Ref.get(sessions)).get(key);
2365
+ yield* releaseEntry({
2366
+ providerSessionId: input.providerSessionId,
2367
+ reason: input.reason,
2368
+ ...(input.detail === undefined ? {} : { detail: input.detail }),
2369
+ keepIfBusyOtherThan: input.ownProviderThreadIds,
2370
+ });
2371
+ // Identity, not presence: open can install a replacement under the same id meanwhile.
2372
+ return before !== undefined && (yield* Ref.get(sessions)).get(key) !== before;
2373
+ }),
2374
+ otherWorkingThreads: Effect.fn("ProviderSessionManager.otherWorkingThreads")(
2375
+ function* (input) {
2376
+ const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId));
2377
+ if (entry === undefined || !entry.supportsMultipleProviderThreads) return [];
2378
+ return yield* Effect.filter(
2379
+ [...entry.attachedThreadIds].filter((threadId) => threadId !== input.threadId),
2380
+ (threadId) =>
2381
+ // Runs, not the shell's activeRunId: the shell omits silent runs such as idle
2382
+ // compaction. Background work outlives its run and still needs the process.
2383
+ Effect.zipWith(
2384
+ projectionStore.getThreadRecords(threadId, ["runs"]),
2385
+ projectionStore.getThreadShell(threadId),
2386
+ ({ runs }, shell) =>
2387
+ runs.some((run) => WORKING_RUN_STATUSES.has(run.status)) ||
2388
+ (shell?.pendingBackgroundTasks?.length ?? 0) > 0,
2389
+ ).pipe(
2390
+ // An unreadable thread may still be working; never kill it unseen.
2391
+ Effect.orElseSucceed(() => true),
2392
+ ),
2393
+ );
2394
+ },
2395
+ ),
2396
detach: Effect.fn("ProviderSessionManager.detach")(
2397
function* (input) {
2398
const key = sessionKey(input.providerSessionId);
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+2
@@ -284,6 +284,8 @@ it.effect(
284
closeInstance: () => Effect.void,
285
release: () => Effect.void,
286
detach: () => Effect.void,
287
+ otherWorkingThreads: () => Effect.succeed([]),
288
+ releaseUnlessBusy: () => Effect.succeed(true),
289
}),
290
);
291
const layerControl = ProviderTurnControlService.layer.pipe(
apps/server/src/orchestration-v2/ProviderTurnControlService.ts
+22
-5
@@ -38,6 +38,8 @@ export class ProviderTurnControlError extends Schema.TaggedError<ProviderTurnCon
38
operation: Schema.Literals(["interrupt", "restart", "steer"]),
39
providerTurnId: ProviderTurnId,
40
turnCompleted: Schema.optional(Schema.Boolean),
41
+ /** The provider never confirmed the stop; Force stop is the way out. */
42
+ unresponsive: Schema.optional(Schema.Boolean),
43
cause: Schema.optional(Schema.Defect()),
44
},
45
) {}
@@ -218,11 +220,26 @@ export const layer: Layer.Layer<
220
});
221
}
222
// Settled turns still reach adapters, which alone know whether background work remains.
221
- yield* session.value.interruptTurn({
222
- providerThread: loaded.providerThread,
223
- providerTurnId: loaded.providerTurn.id,
224
- requestRuntimeRestart: true,
225
- });
223
+ yield* session.value
224
+ .interruptTurn({
225
+ providerThread: loaded.providerThread,
226
+ providerTurnId: loaded.providerTurn.id,
227
+ requestRuntimeRestart: true,
228
+ })
229
+ .pipe(
230
+ Effect.catchTags({
231
+ ProviderAdapterInterruptTimeoutError: (cause) =>
232
+ Effect.fail(
233
+ new ProviderTurnControlError({
234
+ threadId: input.threadId,
235
+ operation: "interrupt",
236
+ providerTurnId: input.providerTurnId,
237
+ unresponsive: true,
238
+ cause,
239
+ }),
240
+ ),
241
+ }),
242
+ );
243
// Use real time; ingestion fibers do not advance TestClock and would leave Stop blocked.
244
yield* Effect.gen(function* () {
245
const deadline = (yield* Clock.currentTimeMillis) + 2_000;
apps/server/src/orchestration-v2/ThreadManagementService.test.ts
+173
@@ -27,6 +27,8 @@ import * as Queue from "effect/Queue";
27
import * as Stream from "effect/Stream";
28
import * as TestClock from "effect/testing/TestClock";
29
30
+import { AgentScope } from "@t3tools/shared/AgentScope";
31
+
32
import * as ServerSettings from "../serverSettings.ts";
33
import * as SqlitePersistence from "../persistence/Sqlite.ts";
34
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
@@ -596,3 +598,174 @@ it.effect.each([
598
expect(dispatched).toEqual(settles ? ["thread.settle"] : []);
599
}),
600
);
601
+
602
+it.effect.each([
603
+ {
604
+ name: "kills the thread's own agent and ends its turn",
605
+ others: [] as ReadonlyArray<string>,
606
+ killed: true,
607
+ expected: { outcome: "killed", sharedWithThreadIds: [] },
608
+ teardown: ["release"],
609
+ },
610
+ {
611
+ name: "keeps a process another working thread shares and only ends this turn",
612
+ others: ["thread:thread-management:force-stop:other"],
613
+ killed: false,
614
+ expected: {
615
+ outcome: "shared",
616
+ sharedWithThreadIds: ["thread:thread-management:force-stop:other"],
617
+ },
618
+ teardown: ["detach"],
619
+ },
620
+ {
621
+ name: "still settles a finished run's background work in a shared process",
622
+ others: ["thread:thread-management:force-stop:other"],
623
+ killed: false,
624
+ settledRun: true,
625
+ expected: {
626
+ outcome: "shared",
627
+ sharedWithThreadIds: ["thread:thread-management:force-stop:other"],
628
+ },
629
+ teardown: ["detach"],
630
+ },
631
+ {
632
+ name: "detaches when a sibling starts a turn just before the session is released",
633
+ others: [] as ReadonlyArray<string>,
634
+ killed: false,
635
+ siblingBusy: true,
636
+ expected: { outcome: "shared", sharedWithThreadIds: [] },
637
+ teardown: ["release", "detach"],
638
+ },
639
+ {
640
+ name: "reports a teardown that has not finished in time as still releasing",
641
+ others: [] as ReadonlyArray<string>,
642
+ killed: false,
643
+ releaseHangs: true,
644
+ expected: { outcome: "releasing", sharedWithThreadIds: [] },
645
+ teardown: ["release"],
646
+ },
647
+])("Force stop $name", ({ others, killed, expected, teardown, ...row }) =>
648
+ Effect.gen(function* () {
649
+ const threadId = ThreadId.make("thread:thread-management:force-stop");
650
+ const providerSessionId = ProviderSessionId.make("provider-session:force-stop");
651
+ const providerThreadId = ProviderThreadId.make("provider-thread:force-stop");
652
+ const runId = RunId.make("run:thread-management:force-stop");
653
+ const dispatched: Array<OrchestrationV2Command | { readonly type: string }> = [];
654
+ const teardowns: Array<string> = [];
655
+ const tornDown = yield* Deferred.make<void>();
656
+ const layerTest = ThreadManagementService.layer.pipe(
657
+ Layer.provide(
658
+ Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
659
+ isLive: () => Effect.succeed(true),
660
+ otherWorkingThreads: () => Effect.succeed(others.map((id) => ThreadId.make(id))),
661
+ releaseUnlessBusy: () =>
662
+ Effect.sync(() => teardowns.push("release")).pipe(
663
+ Effect.andThen(
664
+ "releaseHangs" in row ? Effect.never : Effect.succeed(!("siblingBusy" in row)),
665
+ ),
666
+ ),
667
+ detach: () =>
668
+ Effect.sync(() => teardowns.push("detach")).pipe(
669
+ Effect.andThen(Deferred.succeed(tornDown, undefined)),
670
+ Effect.asVoid,
671
+ ),
672
+ }),
673
+ ),
674
+ Layer.provide(
675
+ Layer.mock(Orchestrator.OrchestratorV2)({
676
+ getThreadRecords: () =>
677
+ Effect.succeed({
678
+ thread: { id: threadId, deletedAt: null },
679
+ runs: [
680
+ {
681
+ id: runId,
682
+ ordinal: 1,
683
+ status: "settledRun" in row ? "completed" : "running",
684
+ activeAttemptId: "attempt",
685
+ },
686
+ // A queued follow-up neither supersedes the request nor becomes the target.
687
+ { id: RunId.make("run:queued-follow-up"), ordinal: 2, status: "queued" },
688
+ ],
689
+ providerThreads: [{ id: providerThreadId, appThreadId: threadId, providerSessionId }],
690
+ providerTurns: [
691
+ {
692
+ id: "provider-turn:force-stop",
693
+ providerThreadId,
694
+ runAttemptId: "attempt",
695
+ status: "settledRun" in row ? "completed" : "running",
696
+ },
697
+ ],
698
+ } as unknown as OrchestrationV2ThreadProjection),
699
+ dispatch: (command) =>
700
+ Effect.sync(() => {
701
+ dispatched.push(command);
702
+ return { sequence: 1, storedEvents: [] } as never;
703
+ }),
704
+ }),
705
+ ),
706
+ Layer.provide(
707
+ Layer.succeed(AgentScope, {
708
+ ...AgentScope.defaultValue(),
709
+ kill: () => Effect.succeed(killed),
710
+ }),
711
+ ),
712
+ Layer.provide(ServerSettings.layerTest()),
713
+ Layer.provide(layerReceiptsTest),
714
+ );
715
+ yield* Effect.gen(function* () {
716
+ const service = yield* ThreadManagementService.ThreadManagementService;
717
+ const stopping = yield* service.forceStopThread({ threadId, runId }).pipe(Effect.forkChild);
718
+ yield* TestClock.adjust("10 seconds");
719
+ expect(yield* Fiber.join(stopping)).toEqual(expected);
720
+ expect(dispatched).toMatchObject([
721
+ {
722
+ type: "thread.background-work.settle",
723
+ providerThreadId,
724
+ reason: "Force stopped by user",
725
+ },
726
+ ]);
727
+ if (teardown.at(-1) === "detach") yield* Deferred.await(tornDown);
728
+ expect(teardowns).toEqual(teardown);
729
+ }).pipe(Effect.provide(layerTest));
730
+ }),
731
+);
732
+
733
+it.effect("Force stop refuses once a newer run has started, before killing anything", () =>
734
+ Effect.gen(function* () {
735
+ const threadId = ThreadId.make("thread:thread-management:force-stop-superseded");
736
+ let killed = false;
737
+ const layerTest = ThreadManagementService.layer.pipe(
738
+ Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
739
+ Layer.provide(
740
+ Layer.mock(Orchestrator.OrchestratorV2)({
741
+ getThreadRecords: () =>
742
+ Effect.succeed({
743
+ thread: { id: threadId, deletedAt: null },
744
+ runs: [
745
+ { id: RunId.make("run:stopped"), ordinal: 1, status: "running" },
746
+ { id: RunId.make("run:newer"), ordinal: 2, status: "running" },
747
+ ],
748
+ providerThreads: [],
749
+ providerTurns: [],
750
+ } as unknown as OrchestrationV2ThreadProjection),
751
+ }),
752
+ ),
753
+ Layer.provide(
754
+ Layer.succeed(AgentScope, {
755
+ ...AgentScope.defaultValue(),
756
+ kill: () => Effect.sync(() => (killed = true)),
757
+ }),
758
+ ),
759
+ Layer.provide(ServerSettings.layerTest()),
760
+ Layer.provide(layerReceiptsTest),
761
+ );
762
+ const error = yield* Effect.gen(function* () {
763
+ const service = yield* ThreadManagementService.ThreadManagementService;
764
+ return yield* Effect.flip(
765
+ service.forceStopThread({ threadId, runId: RunId.make("run:stopped") }),
766
+ );
767
+ }).pipe(Effect.provide(layerTest));
768
+ expect(error.reason).toBe("superseded");
769
+ expect(killed).toBe(false);
770
+ }),
771
+);
apps/server/src/orchestration-v2/ThreadManagementService.ts
+166
@@ -14,6 +14,9 @@ import {
14
MessageId,
15
OrchestrationV2StopWorkflowError,
16
type OrchestrationV2StopWorkflowInput,
17
+ OrchestrationV2ForceStopThreadError,
18
+ type OrchestrationV2ForceStopThreadInput,
19
+ type OrchestrationV2ForceStopThreadResult,
20
type ModelSelection,
21
type OrchestrationV2Actor,
22
type OrchestrationV2Command,
@@ -30,6 +33,7 @@ import {
33
ProjectId,
34
PROVIDER_SEND_TURN_MAX_FILE_BYTES,
35
ProviderInstanceId,
36
+ type ProviderSessionId,
37
resolveProjectProviderInstanceEnabled,
38
RunId,
39
type ScheduledTaskId,
@@ -38,10 +42,13 @@ import {
42
} from "@t3tools/contracts";
43
import { delegatedTaskProgress } from "@t3tools/provider-core/server/subagentProjection";
44
import { threadTranscriptHeader } from "@t3tools/shared/threadTranscript";
45
+import { AgentScope } from "@t3tools/shared/AgentScope";
46
+import * as Clock from "effect/Clock";
47
import * as Context from "effect/Context";
48
import * as DateTime from "effect/DateTime";
49
import * as Duration from "effect/Duration";
50
import * as Effect from "effect/Effect";
51
+import * as Fiber from "effect/Fiber";
52
import * as Layer from "effect/Layer";
53
import * as Option from "effect/Option";
54
import * as Ref from "effect/Ref";
@@ -454,6 +461,13 @@ export interface ThreadManagementServiceShape {
461
readonly stopWorkflow: (
462
input: OrchestrationV2StopWorkflowInput,
463
) => Effect.Effect<void, OrchestrationV2StopWorkflowError>;
464
+ /**
465
+ * Kills the thread's agent for a provider that no longer responds, then ends
466
+ * its turn as interrupted. Never kills a process other working threads share.
467
+ */
468
+ readonly forceStopThread: (
469
+ input: OrchestrationV2ForceStopThreadInput,
470
+ ) => Effect.Effect<OrchestrationV2ForceStopThreadResult, OrchestrationV2ForceStopThreadError>;
471
readonly awaitBackgroundTasks: (input: {
472
readonly threadId: ThreadId;
473
readonly taskIds: ReadonlyArray<string>;
@@ -547,12 +561,16 @@ function latestSteerableRun(
561
}
562
563
const SETTLE_AFTER_RUN_WAIT_MS = 24 * 60 * 60 * 1_000;
564
+const FORCE_STOP_REASON = "Force stopped by user";
565
+const FORCE_STOP_RELEASE_WAIT = "10 seconds";
566
+const isForceStopThreadError = Schema.is(OrchestrationV2ForceStopThreadError);
567
568
const make = Effect.gen(function* () {
569
const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor;
570
const orchestrator = yield* Orchestrator.OrchestratorV2;
571
const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
572
const layerScope = yield* Effect.scope;
573
+ const agentScope = yield* AgentScope;
574
const legacyImporter = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter;
575
const serverSettings = yield* ServerSettings.ServerSettingsService;
576
const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
@@ -1072,6 +1090,153 @@ const make = Effect.gen(function* () {
1090
return { type: "interrupt_requested", run: interruptibleRun, dispatch } as const;
1091
});
1092
1093
+ const forceStopThread: ThreadManagementServiceShape["forceStopThread"] = Effect.fn(
1094
+ "ThreadManagementService.forceStopThread",
1095
+ )(
1096
+ function* (input) {
1097
+ // Under the thread lock no new run can be created between the check and the kill. The
1098
+ // settlement below dispatches, which takes that lock itself, so it runs after release.
1099
+ const { records, target, killed } = yield* threadDispatch.withLock(
1100
+ input.threadId,
1101
+ Effect.gen(function* () {
1102
+ const records = yield* orchestrator.getThreadRecords(input.threadId, [
1103
+ "runs",
1104
+ "providerThreads",
1105
+ "providerTurns",
1106
+ ]);
1107
+ // The run Force stop acts on: the active one, else the latest that started. Queued
1108
+ // follow-ups never ran, so they neither count as targets nor supersede the request.
1109
+ const target =
1110
+ latestActiveRun(records) ??
1111
+ records.runs
1112
+ .filter((run) => run.status !== "queued")
1113
+ .reduce<OrchestrationV2Run | undefined>(
1114
+ (current, run) =>
1115
+ current === undefined || run.ordinal > current.ordinal ? run : current,
1116
+ undefined,
1117
+ );
1118
+ if (input.runId !== undefined && input.runId !== target?.id) {
1119
+ return yield* new OrchestrationV2ForceStopThreadError({
1120
+ threadId: input.threadId,
1121
+ reason: "superseded",
1122
+ });
1123
+ }
1124
+ // Shared scopes are never tracked per thread, so this only kills the thread's own agent.
1125
+ const killed = yield* agentScope.kill(input.threadId);
1126
+ return { records, target, killed };
1127
+ }),
1128
+ );
1129
+ const sessionIds = new Set(
1130
+ records.providerThreads.flatMap((providerThread) =>
1131
+ providerThread.appThreadId === input.threadId && providerThread.providerSessionId !== null
1132
+ ? [providerThread.providerSessionId]
1133
+ : [],
1134
+ ),
1135
+ );
1136
+ // A run that already ended may still hold background work; settling it is a no-op otherwise.
1137
+ const run = target;
1138
+ if (run !== undefined) {
1139
+ const active = isActiveRun(run);
1140
+ const providerTurn = records.providerTurns.findLast(
1141
+ (turn) =>
1142
+ turn.runAttemptId === run.activeAttemptId && (!active || turn.status === "running"),
1143
+ );
1144
+ const commandId = CommandId.make(
1145
+ `server:force-stop:${run.id}:${yield* Clock.currentTimeMillis}`,
1146
+ );
1147
+ yield* providerTurn === undefined && !active
1148
+ ? Effect.void
1149
+ : providerTurn === undefined
1150
+ ? orchestrator.dispatch({
1151
+ type: "run.interrupt",
1152
+ commandId,
1153
+ threadId: input.threadId,
1154
+ runId: run.id,
1155
+ reason: FORCE_STOP_REASON,
1156
+ })
1157
+ : orchestrator.dispatch({
1158
+ type: "thread.background-work.settle",
1159
+ commandId,
1160
+ threadId: input.threadId,
1161
+ providerThreadId: providerTurn.providerThreadId,
1162
+ providerTurnId: providerTurn.id,
1163
+ reason: FORCE_STOP_REASON,
1164
+ });
1165
+ }
1166
+ const ownProviderThreadIds = records.providerThreads.flatMap((providerThread) =>
1167
+ providerThread.appThreadId === input.threadId ? [providerThread.id] : [],
1168
+ );
1169
+ const detach = (providerSessionId: ProviderSessionId) =>
1170
+ sessions
1171
+ .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON })
1172
+ .pipe(
1173
+ Effect.catchCause((cause) =>
1174
+ Effect.logWarning("orchestration-v2.force-stop-detach-failed", {
1175
+ threadId: input.threadId,
1176
+ cause,
1177
+ }),
1178
+ ),
1179
+ Effect.forkIn(layerScope),
1180
+ );
1181
+ let shared = false;
1182
+ const sharedWith = new Set<ThreadId>();
1183
+ const releases: Array<{
1184
+ readonly providerSessionId: ProviderSessionId;
1185
+ readonly fiber: Fiber.Fiber<boolean, ProviderSessionManager.ProviderSessionManagerV2Error>;
1186
+ }> = [];
1187
+ for (const providerSessionId of sessionIds) {
1188
+ if (!(yield* sessions.isLive(providerSessionId))) continue;
1189
+ // Decided just before teardown, so the reply matches what happens to the process.
1190
+ const others = yield* sessions.otherWorkingThreads({
1191
+ providerSessionId,
1192
+ threadId: input.threadId,
1193
+ });
1194
+ for (const threadId of others) sharedWith.add(threadId);
1195
+ if (others.length > 0) {
1196
+ shared = true;
1197
+ yield* detach(providerSessionId);
1198
+ continue;
1199
+ }
1200
+ // The session manager re-checks atomically with turn starts and keeps a session a
1201
+ // sibling began using in the meantime.
1202
+ const fiber = yield* sessions
1203
+ .releaseUnlessBusy({
1204
+ providerSessionId,
1205
+ ownProviderThreadIds,
1206
+ reason: "manual_shutdown",
1207
+ detail: FORCE_STOP_REASON,
1208
+ })
1209
+ .pipe(Effect.forkIn(layerScope));
1210
+ releases.push({ providerSessionId, fiber });
1211
+ }
1212
+ // Teardown waits on the provider's own exit paths; past this bound it keeps going on its own.
1213
+ let releasing = false;
1214
+ for (const release of releases) {
1215
+ const released = yield* Fiber.join(release.fiber).pipe(
1216
+ Effect.timeoutOption(FORCE_STOP_RELEASE_WAIT),
1217
+ );
1218
+ if (Option.isNone(released)) {
1219
+ releasing = true;
1220
+ } else if (!released.value) {
1221
+ shared = true;
1222
+ yield* detach(release.providerSessionId);
1223
+ }
1224
+ }
1225
+ return {
1226
+ outcome: killed ? "killed" : shared ? "shared" : releasing ? "releasing" : "released",
1227
+ sharedWithThreadIds: [...sharedWith],
1228
+ } as const;
1229
+ },
1230
+ (effect, input) =>
1231
+ effect.pipe(
1232
+ Effect.mapError((cause) =>
1233
+ isForceStopThreadError(cause)
1234
+ ? cause
1235
+ : new OrchestrationV2ForceStopThreadError({ threadId: input.threadId, cause }),
1236
+ ),
1237
+ ),
1238
+ );
1239
+
1240
const stopWorkflow: ThreadManagementServiceShape["stopWorkflow"] = Effect.fn(
1241
"ThreadManagementService.stopWorkflow",
1242
)(
@@ -1354,6 +1519,7 @@ const make = Effect.gen(function* () {
1519
interruptThread,
1520
stopDelegatedTasks,
1521
stopWorkflow,
1522
+ forceStopThread,
1523
awaitBackgroundTasks,
1524
getThreadEventSequence: orchestrator.getThreadEventSequence,
1525
recoverDelegatedTask: orchestrator.recoverDelegatedTask,
apps/server/src/orchestration-v2/ThreadStop.test.ts
+112
@@ -1552,3 +1552,115 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)
1552
);
1553
}).pipe(Effect.provide(layerTest)),
1554
);
1555
+
1556
+it.effect("an unanswered Stop is flagged until Force stop ends the run with its own reason", () =>
1557
+ Effect.gen(function* () {
1558
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1559
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
1560
+ const threadId = ThreadId.make("thread:force-stop");
1561
+ yield* createWatchingThread(threadId, 6);
1562
+ yield* send(threadId, "work", "start_immediately");
1563
+ const now = yield* DateTime.now;
1564
+ const started = yield* orchestrator.getThreadProjection(threadId);
1565
+ const run = started.runs[0]!;
1566
+ const providerThreadId = run.providerThreadId!;
1567
+ const providerTurnId = ProviderTurnId.make("provider-turn:force-stop");
1568
+ const apply = (event: Omit<OrchestrationV2DomainEvent, "id" | "threadId">) =>
1569
+ projections.apply({
1570
+ ...event,
1571
+ id: EventId.make(`event:force-stop:${event.type}`),
1572
+ threadId,
1573
+ } as OrchestrationV2DomainEvent);
1574
+ const attempt = started.attempts[0]!;
1575
+ const attemptId = attempt.id;
1576
+ yield* apply({
1577
+ type: "run.updated",
1578
+ runId: run.id,
1579
+ occurredAt: now,
1580
+ payload: { ...run, status: "running", startedAt: now, activeAttemptId: attemptId },
1581
+ });
1582
+ yield* apply({
1583
+ type: "run-attempt.updated",
1584
+ occurredAt: now,
1585
+ payload: { ...attempt, status: "running", startedAt: now },
1586
+ });
1587
+ yield* apply({
1588
+ type: "provider-turn.updated",
1589
+ occurredAt: now,
1590
+ payload: {
1591
+ id: providerTurnId,
1592
+ providerThreadId,
1593
+ nodeId: run.rootNodeId!,
1594
+ runAttemptId: attemptId,
1595
+ nativeTurnRef: null,
1596
+ ordinal: 1,
1597
+ status: "running",
1598
+ startedAt: now,
1599
+ completedAt: null,
1600
+ },
1601
+ });
1602
+ yield* apply({
1603
+ type: "turn-item.updated",
1604
+ runId: run.id,
1605
+ occurredAt: now,
1606
+ payload: {
1607
+ id: TurnItemId.make("turn-item:force-stop:interrupt-request"),
1608
+ threadId,
1609
+ runId: run.id,
1610
+ nodeId: run.rootNodeId!,
1611
+ providerThreadId,
1612
+ providerTurnId,
1613
+ nativeItemRef: null,
1614
+ parentItemId: null,
1615
+ ordinal: 1_000,
1616
+ status: "completed",
1617
+ title: "Interrupt requested",
1618
+ startedAt: now,
1619
+ completedAt: now,
1620
+ updatedAt: now,
1621
+ type: "run_interrupt_request",
1622
+ message: "Stopped by user",
1623
+ },
1624
+ });
1625
+ const target = { threadId, providerThreadId, providerTurnId };
1626
+ const request = Effect.map(orchestrator.getThreadProjection(threadId), (projection) =>
1627
+ projection.turnItems.find((item) => item.type === "run_interrupt_request"),
1628
+ );
1629
+
1630
+ yield* orchestrator.dispatch({
1631
+ ...target,
1632
+ type: "thread.interrupt.unresponsive",
1633
+ commandId: CommandId.make("force-stop:unresponsive"),
1634
+ });
1635
+ const flagged = yield* request;
1636
+ assert.isTrue(
1637
+ flagged?.type === "run_interrupt_request" && flagged.unresponsiveAt !== undefined,
1638
+ );
1639
+ assert.deepEqual((yield* threadState(threadId)).runs, ["running"]);
1640
+
1641
+ yield* orchestrator.dispatch({
1642
+ ...target,
1643
+ type: "thread.background-work.settle",
1644
+ commandId: CommandId.make("force-stop:settle"),
1645
+ reason: "Force stopped by user",
1646
+ });
1647
+ const settled = yield* orchestrator.getThreadProjection(threadId);
1648
+ assert.deepEqual(
1649
+ settled.runs.map((candidate) => candidate.status),
1650
+ ["interrupted"],
1651
+ );
1652
+ const result = settled.turnItems.find((item) => item.type === "run_interrupt_result");
1653
+ assert.equal(
1654
+ result?.type === "run_interrupt_result" ? result.message : undefined,
1655
+ "Force stopped by user",
1656
+ );
1657
+
1658
+ // A provider confirming late changes nothing once the run has ended.
1659
+ const late = yield* orchestrator.dispatch({
1660
+ ...target,
1661
+ type: "thread.interrupt.unresponsive",
1662
+ commandId: CommandId.make("force-stop:late-unresponsive"),
1663
+ });
1664
+ assert.lengthOf(late.storedEvents, 0);
1665
+ }).pipe(Effect.provide(layerTest)),
1666
+);
apps/server/src/process/agentScope.test.ts
+32
@@ -110,6 +110,8 @@ const withFakeSystemd = <A, E>(
110
...fileSystem,
111
readFileString: (path, encoding) =>
112
fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
113
+ writeFileString: (path, data, options) =>
114
+ fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
115
}),
116
);
117
const { stateDir } = yield* ServerConfig.ServerConfig;
@@ -213,6 +215,36 @@ describe("AgentScope service", () => {
215
),
216
);
217
218
+ it.effect.each([{ oomPolicy: true }, { oomPolicy: false }])(
219
+ "Force stop kills a thread's own scope but never a shared Codex scope (OOM policy $oomPolicy)",
220
+ ({ oomPolicy }) =>
221
+ withFakeSystemd(
222
+ ({ scope, units, calls, cgroupRoot }) =>
223
+ Effect.gen(function* () {
224
+ const agent = yield* scope.wrap({
225
+ command: "agent",
226
+ args: [],
227
+ name: "t",
228
+ threadId: "a",
229
+ });
230
+ exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
231
+ const codex = yield* launchCodex(scope, "b");
232
+ exitedAgent(units, cgroupRoot, unitOf(codex), { populated: true, oomKills: 0 });
233
+
234
+ expect(yield* scope.kill("a")).toBe(true);
235
+ expect(
236
+ NodeFS.readFileSync(NodePath.join(cgroupRoot, unitOf(agent), "cgroup.kill"), "utf8"),
237
+ ).toBe("1");
238
+ expect(yield* scope.kill("b")).toBe(false);
239
+ expect(NodeFS.existsSync(NodePath.join(cgroupRoot, unitOf(codex), "cgroup.kill"))).toBe(
240
+ false,
241
+ );
242
+ expect(calls.filter(([, verb]) => verb === "kill")).toEqual([]);
243
+ }),
244
+ { oomPolicy },
245
+ ),
246
+ );
247
+
248
it.effect("keeps a running scope while hundreds of other threads launch", () =>
249
withFakeSystemd(({ scope, units }) =>
250
Effect.gen(function* () {
apps/server/src/process/agentScope.ts
+41
-3
@@ -215,7 +215,8 @@ export const make = Effect.gen(function* () {
215
}),
216
);
217
218
- const scopeByThread = new Map<string, { unit: string; oomKilled: boolean }>();
218
+ // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd.
219
+ const scopeByThread = new Map<string, { unit: string; oomKilled: boolean; shared: boolean }>();
220
221
const readScope = Effect.fn("agentScope.readScope")(function* (systemctl: string, unit: string) {
222
const shown = yield* run(systemctl, [
@@ -277,10 +278,11 @@ export const make = Effect.gen(function* () {
278
systemctl: string,
279
threadId: string,
280
unit: string | undefined,
281
+ shared = false,
282
) {
283
const previous = scopeByThread.get(threadId);
284
scopeByThread.delete(threadId);
283
- if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false });
285
+ if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared });
286
if (previous !== undefined) yield* forget(systemctl, previous.unit);
287
if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl);
288
});
@@ -340,6 +342,40 @@ export const make = Effect.gen(function* () {
342
}
343
});
344
345
+ const kill: AgentScopeShape["kill"] = Effect.fn("AgentScope.kill")(function* (threadId) {
346
+ const tracked = scopeByThread.get(threadId);
347
+ if (tracked === undefined || tracked.shared) return false;
348
+ const scope = yield* systemd;
349
+ if (scope === undefined) return false;
350
+ const shown = yield* run(scope.systemctl, [
351
+ "--user",
352
+ "show",
353
+ tracked.unit,
354
+ "--property=ActiveState,ControlGroup",
355
+ ]);
356
+ if (shown._tag === "None" || shown.value.code !== 0) return false;
357
+ const values = parseKeyValues(shown.value.stdout);
358
+ if (values.get("ActiveState") !== "active") return false;
359
+ const controlGroup = values.get("ControlGroup") ?? "";
360
+ // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback.
361
+ const killedByCgroup =
362
+ controlGroup.length > 0 &&
363
+ (yield* fileSystem
364
+ .writeFileString(path.join("/sys/fs/cgroup", controlGroup, "cgroup.kill"), "1")
365
+ .pipe(
366
+ Effect.as(true),
367
+ Effect.orElseSucceed(() => false),
368
+ ));
369
+ if (killedByCgroup) return true;
370
+ const killed = yield* run(scope.systemctl, [
371
+ "--user",
372
+ "kill",
373
+ "--signal=SIGKILL",
374
+ tracked.unit,
375
+ ]);
376
+ return killed._tag === "Some" && killed.value.code === 0;
377
+ });
378
+
379
const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) {
380
if (!scopeByThread.has(threadId)) return;
381
const scope = yield* systemd;
@@ -365,7 +401,8 @@ export const make = Effect.gen(function* () {
401
// Clear the launch thread's previous scope instead.
402
const continuing = shared === true && scope.supportsOomPolicy;
403
const owned = threadId !== undefined && !continuing;
368
- if (threadId !== undefined) yield* track(scope.systemctl, threadId, owned ? unit : undefined);
404
+ if (threadId !== undefined)
405
+ yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true);
406
const systemdScope: AgentSystemdScope = {
407
systemdRun: scope.systemdRun,
408
runtimeDir: scope.runtimeDir,
@@ -385,6 +422,7 @@ export const make = Effect.gen(function* () {
422
}),
423
oomKilled,
424
stop,
425
+ kill,
426
clear,
427
} satisfies AgentScopeShape;
428
});
apps/server/src/relay/AgentAwarenessRelay.test.ts
+1
@@ -219,6 +219,7 @@ const makeTestRelay = Effect.fnUntraced(function* (
219
settleThread: unused,
220
interruptThread: unused,
221
stopWorkflow: () => Effect.die("unused stopWorkflow"),
222
+ forceStopThread: () => Effect.die("unused forceStopThread"),
223
awaitBackgroundTasks: unused,
224
stopDelegatedTasks: unused,
225
getThreadEventSequence: unused,
apps/server/src/ws.ts
+2
@@ -1914,6 +1914,8 @@ export const layerWsRpc = (
1914
),
1915
),
1916
[ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: (input) => threadManagement.stopWorkflow(input),
1917
+ [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: (input) =>
1918
+ threadManagement.forceStopThread(input),
1919
[ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: (input) =>
1920
readWorkflowScript({ scriptPath: input.scriptPath }),
1921
[ORCHESTRATION_V2_WS_METHODS.getTurnItem]: (input) =>
apps/web/src/components/ChatView.tsx
+5
-7
@@ -2349,7 +2349,8 @@ export default function ChatView(props: ChatViewProps) {
2349
const {
2350
acknowledgeActiveThreadWoke,
2351
activeBranchMismatchKey,
2352
- backgroundWorkNow,
2352
+ attentionNow,
2353
+ workNow,
2354
childInputNow,
2355
goalNow,
2356
handleSwitchCheckoutToThread,
@@ -2993,10 +2994,7 @@ export default function ChatView(props: ChatViewProps) {
2994
const onExpandTimelineImageStable = useStableCallback(onExpandTimelineImage);
2995
const shownThreadSyncPhase = activeEnvironmentUnavailable ? null : delayedThreadSyncPhase;
2996
const hasThreadNow =
2996
- childInputNow !== null ||
2997
- backgroundWorkNow !== null ||
2998
- goalNow !== null ||
2999
- shownThreadSyncPhase !== null;
2997
+ attentionNow !== null || workNow !== null || goalNow !== null || shownThreadSyncPhase !== null;
2998
const queuedRunCount = serverProjection ? getUserQueuedThreadRuns(serverProjection).length : 0;
2999
// The queue control is mounted for every server thread but joins the footer only while it shows
3000
// something, by its own rule: an optimistic message stops counting once the server has it.
@@ -3035,8 +3033,8 @@ export default function ChatView(props: ChatViewProps) {
3033
)}
3034
<ThreadNow
3035
key={routeThreadKey}
3038
- attention={childInputNow}
3039
- work={backgroundWorkNow}
3036
+ attention={attentionNow}
3037
+ work={workNow}
3038
syncPhase={shownThreadSyncPhase}
3039
goal={goalNow}
3040
/>
apps/web/src/components/chat/ThreadActivityBanners.tsx
+43
@@ -7,17 +7,20 @@ import type { EnvironmentThreadShell, ThreadStructure } from "@t3tools/client-ru
7
import {
8
presentPendingBackgroundWork,
9
presentProviderGoal,
10
+ type RunStopPresentation,
11
} from "@t3tools/client-runtime/state/thread-execution";
12
import { childThreadInputsTitle } from "@t3tools/client-runtime/state/threads";
13
import { formatDuration } from "@t3tools/shared/orchestrationTiming";
14
import * as DateTime from "effect/DateTime";
15
import {
16
CircleDashedIcon,
17
+ CircleStopIcon,
18
GitBranchIcon,
19
MessageCircleIcon,
20
ShieldQuestionIcon,
21
TargetIcon,
22
TerminalIcon,
23
+ TriangleAlertIcon,
24
} from "lucide-react";
25
import { Fragment } from "react";
26
import { tickTextContent } from "~/lib/tickTextContent";
@@ -105,6 +108,46 @@ export function goalNowItem(input: {
108
};
109
}
110
111
+/** A Stop the provider has not finished; Force stop once it stops answering. */
112
+export function runStopNowItem(input: {
113
+ readonly threadId: ThreadId;
114
+ readonly stop: RunStopPresentation;
115
+ readonly canForceStop: boolean;
116
+ readonly forceStopping: boolean;
117
+ readonly onForceStop: () => void;
118
+}): ThreadNowItem {
119
+ const unresponsive = input.stop.state === "unresponsive";
120
+ const requestedAtMs = Date.parse(input.stop.requestedAt);
121
+ return {
122
+ id: `run-stop:${input.threadId}`,
123
+ icon: unresponsive ? (
124
+ <TriangleAlertIcon className="text-warning-foreground" />
125
+ ) : (
126
+ <CircleStopIcon />
127
+ ),
128
+ title: unresponsive ? "Agent not responding to Stop" : "Stopping",
129
+ description: unresponsive
130
+ ? "It may be short on memory. Force stop kills its processes."
131
+ : undefined,
132
+ actions: (
133
+ <>
134
+ <span
135
+ ref={tickTextContent(
136
+ (nowMs) => formatDuration(Math.max(1_000, nowMs - requestedAtMs)),
137
+ true,
138
+ )}
139
+ className="tabular-nums"
140
+ />
141
+ {unresponsive && input.canForceStop ? (
142
+ <InlineButton disabled={input.forceStopping} onClick={input.onForceStop}>
143
+ {input.forceStopping ? "Force stopping..." : "Force stop"}
144
+ </InlineButton>
145
+ ) : null}
146
+ </>
147
+ ),
148
+ };
149
+}
150
+
151
/** Background commands and subagents, for the end of the timeline. */
152
export function backgroundWorkNowItem(input: {
153
readonly threadId: ThreadId;
apps/web/src/components/chat/useForceStopThread.ts
new
+59
@@ -0,0 +1,59 @@
1
+import { useAtomValue } from "@effect/atom-react";
2
+import type { EnvironmentId, RunId, ThreadId } from "@t3tools/contracts";
3
+import { useState } from "react";
4
+
5
+import { useServerConfigs } from "../../state/entities";
6
+import { orchestrationEnvironment } from "../../state/orchestration";
7
+import {
8
+ isAtomCommandInterrupted,
9
+ squashAtomCommandFailure,
10
+} from "@t3tools/client-runtime/state/runtime";
11
+import { useAtomCommand } from "../../state/use-atom-command";
12
+import { toastManager } from "../ui/toast";
13
+
14
+/** Force stop for an agent that no longer answers Stop; shared by every place that offers it. */
15
+export function useForceStopThread(environmentId: EnvironmentId) {
16
+ const forceStop = useAtomCommand(orchestrationEnvironment.forceStopThread);
17
+ const permitted = useAtomValue(
18
+ orchestrationEnvironment.forceStopThread.permissionAtom(environmentId),
19
+ );
20
+ const supported =
21
+ useServerConfigs().get(environmentId)?.environment.capabilities.threadForceStop === true;
22
+ const canForceStop = permitted && supported;
23
+ const [stoppingThreadId, setStoppingThreadId] = useState<ThreadId | null>(null);
24
+ /** Pass the run that ignored Stop, so a Force stop left open cannot hit a newer run. */
25
+ const forceStopThread = async (threadId: ThreadId, runId?: RunId) => {
26
+ if (!canForceStop || stoppingThreadId !== null) return;
27
+ setStoppingThreadId(threadId);
28
+ const result = await forceStop({
29
+ environmentId,
30
+ input: runId === undefined ? { threadId } : { threadId, runId },
31
+ });
32
+ setStoppingThreadId(null);
33
+ if (result._tag === "Failure") {
34
+ if (isAtomCommandInterrupted(result)) return;
35
+ const error = squashAtomCommandFailure(result);
36
+ toastManager.add({
37
+ type: "error",
38
+ title: "Could not force stop the agent",
39
+ description: error instanceof Error ? error.message : undefined,
40
+ });
41
+ return;
42
+ }
43
+ const others = result.value.sharedWithThreadIds.length;
44
+ if (result.value.outcome === "releasing") {
45
+ toastManager.add({
46
+ type: "info",
47
+ title: "Turn stopped; the agent is still shutting down",
48
+ });
49
+ }
50
+ if (result.value.outcome === "shared") {
51
+ toastManager.add({
52
+ type: "info",
53
+ title: "Turn stopped; the agent process kept running",
54
+ description: `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`,
55
+ });
56
+ }
57
+ };
58
+ return { canForceStop, stoppingThreadId, forceStopThread };
59
+}
apps/web/src/components/chat/useThreadLifecycle.tsx
+33
-2
@@ -58,9 +58,22 @@ import { appAtomRegistry } from "../../rpc/atomRegistry";
58
import { readEnvironmentScope } from "../../state/session";
59
import { useAtomCommand } from "../../state/use-atom-command";
60
import { useOrchestrationCommand } from "../../state/use-orchestration-command";
61
-import { backgroundWorkNowItem, childInputNowItem, goalNowItem } from "./ThreadActivityBanners";
61
+import {
62
+ backgroundWorkNowItem,
63
+ childInputNowItem,
64
+ goalNowItem,
65
+ runStopNowItem,
66
+} from "./ThreadActivityBanners";
67
+import { useForceStopThread } from "./useForceStopThread";
68
+import { useAtomValue } from "@effect/atom-react";
69
+import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution";
70
+import { Atom } from "effect/reactivity";
71
+import { environmentThreadDetails } from "../../state/threads";
72
import type { useLocalDispatchState } from "./useLocalDispatchState";
73
const RELATED_THREAD_SHELL_WAIT_MS = 1_000;
74
+const NO_RUN_STOP_ATOM = Atom.make<RunStopPresentation | null>(null).pipe(
75
+ Atom.withLabel("web-thread-run-stop:empty"),
76
+);
77
78
type PendingThreadRequests = ReturnType<typeof usePendingThreadRequests>;
79
type LocalDispatchState = ReturnType<typeof useLocalDispatchState>;
@@ -509,6 +522,22 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) {
522
})
523
: null;
524
525
+ const forceStop = useForceStopThread(environmentId);
526
+ const anyRunStop = useAtomValue(
527
+ activeThreadRef ? environmentThreadDetails.runStopAtom(activeThreadRef) : NO_RUN_STOP_ATOM,
528
+ );
529
+ // A background Stop matters only while that work is still pending.
530
+ const runStop = anyRunStop?.background && activeBackgroundTasks.length === 0 ? null : anyRunStop;
531
+ const runStopNow =
532
+ activeThread && runStop !== null
533
+ ? runStopNowItem({
534
+ threadId: activeThread.id,
535
+ stop: runStop,
536
+ canForceStop: forceStop.canForceStop,
537
+ forceStopping: forceStop.stoppingThreadId === activeThread.id,
538
+ onForceStop: () => void forceStop.forceStopThread(activeThread.id, runStop.runId),
539
+ })
540
+ : null;
541
const backgroundWorkNow = activeThread
542
? backgroundWorkNowItem({
543
threadId: activeThread.id,
@@ -592,7 +621,9 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) {
621
shortcuts,
622
acknowledgeActiveThreadWoke,
623
activeBranchMismatchKey,
595
- backgroundWorkNow,
624
+ // A Stop that went unanswered needs the user; one in progress is the thread's work.
625
+ attentionNow: runStop?.state === "unresponsive" ? runStopNow : childInputNow,
626
+ workNow: runStop?.state === "stopping" ? runStopNow : backgroundWorkNow,
627
childInputNow,
628
goalNow,
629
handleSwitchCheckoutToThread,
packages/client-runtime/src/state/orchestration.ts
+4
@@ -107,6 +107,10 @@ export function createOrchestrationEnvironmentAtoms<R, E>(
107
label: "environment-data:orchestration:stop-workflow",
108
tag: ORCHESTRATION_V2_WS_METHODS.stopWorkflow,
109
}),
110
+ forceStopThread: createEnvironmentRpcCommand(runtime, {
111
+ label: "environment-data:orchestration:force-stop-thread",
112
+ tag: ORCHESTRATION_V2_WS_METHODS.forceStopThread,
113
+ }),
114
turnItem: createEnvironmentRpcQueryAtomFamily(runtime, {
115
label: "environment-data:orchestration:turn-item",
116
tag: ORCHESTRATION_V2_WS_METHODS.getTurnItem,
packages/client-runtime/src/state/threadDetail.ts
+28
@@ -14,6 +14,7 @@ import {
14
type ThreadQueueWorkflowState,
15
} from "./threadWorkflows.ts";
16
import { deriveThreadTurnSubagents, type ThreadTurnSubagents } from "./threadSubagents.ts";
17
+import { presentRunStop, type RunStopPresentation } from "./threadExecution.ts";
18
import {
19
deriveThreadCheckpointSummaries,
20
type ThreadCheckpointSummary,
@@ -294,6 +295,32 @@ export function createEnvironmentThreadDetailAtoms<E>(
295
return value;
296
}).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-queue:${key}`));
297
});
298
+ const runStopAtomFamily = Atom.family((key: string) => {
299
+ let previous: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems"> | null = null;
300
+ let stop: RunStopPresentation | null = null;
301
+ return Atom.make((get) => {
302
+ const projection = Option.getOrNull(get(threadStateValueAtomFamily(key)).data);
303
+ if (projection === null) {
304
+ previous = null;
305
+ stop = null;
306
+ } else if (
307
+ projection.runs !== previous?.runs ||
308
+ projection.turnItems !== previous?.turnItems
309
+ ) {
310
+ const { runs, turnItems } = projection;
311
+ previous = { runs, turnItems };
312
+ const next = presentRunStop(previous);
313
+ if (
314
+ next?.state !== stop?.state ||
315
+ next?.runId !== stop?.runId ||
316
+ next?.requestedAt !== stop?.requestedAt ||
317
+ next?.background !== stop?.background
318
+ )
319
+ stop = next;
320
+ }
321
+ return stop;
322
+ }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-run-stop:${key}`));
323
+ });
324
const queuedCountAtomFamily = Atom.family((key: string) => {
325
let previous: Pick<OrchestrationV2ThreadProjection, "runs" | "messages"> | null = null;
326
let count = 0;
@@ -435,6 +462,7 @@ export function createEnvironmentThreadDetailAtoms<E>(
462
pendingRequestsAtom: (ref: ScopedThreadRef) => pendingRequestsAtomFamily(threadKey(ref)),
463
queueWorkflowAtom: (ref: ScopedThreadRef) => queueWorkflowAtomFamily(threadKey(ref)),
464
queuedCountAtom: (ref: ScopedThreadRef) => queuedCountAtomFamily(threadKey(ref)),
465
+ runStopAtom: (ref: ScopedThreadRef) => runStopAtomFamily(threadKey(ref)),
466
turnSubagentsAtom: (ref: ScopedThreadRef) => turnSubagentsAtomFamily(threadKey(ref)),
467
checkpointSummariesAtom: (ref: ScopedThreadRef) =>
468
checkpointSummariesAtomFamily(threadKey(ref)),
packages/client-runtime/src/state/threadExecution.test.ts
+69
@@ -25,6 +25,7 @@ import {
25
deriveRunlessWorkStartedAt,
26
deriveThreadActivityRun,
27
deriveThreadRuntime,
28
+ presentRunStop,
29
threadRuntimeHasInterruptibleRun,
30
} from "./threadExecution.ts";
31
@@ -259,6 +260,74 @@ describe("thread execution presentation", () => {
260
});
261
});
262
263
+describe("presentRunStop", () => {
264
+ const signal = (type: "run_interrupt_request" | "run_interrupt_result", runId: RunId) => ({
265
+ id: TurnItemId.make(`${type}:${runId}`),
266
+ threadId: v2Projection.thread.id,
267
+ runId,
268
+ nodeId: null,
269
+ providerThreadId: null,
270
+ providerTurnId: null,
271
+ nativeItemRef: null,
272
+ parentItemId: null,
273
+ ordinal: 1,
274
+ status: type === "run_interrupt_result" ? ("interrupted" as const) : ("completed" as const),
275
+ title: "Interrupt",
276
+ startedAt: now,
277
+ completedAt: now,
278
+ updatedAt: now,
279
+ type,
280
+ message: "Stopped by user",
281
+ });
282
+ const running = run("run-stopping", 1, "running");
283
+ const request = signal("run_interrupt_request", running.id);
284
+
285
+ it.each([
286
+ { name: "no Stop", runs: [running], turnItems: [], state: null },
287
+ {
288
+ name: "a Stop the agent is winding down",
289
+ runs: [running],
290
+ turnItems: [request],
291
+ state: "stopping",
292
+ },
293
+ {
294
+ name: "a Stop the provider never confirmed",
295
+ runs: [running],
296
+ turnItems: [{ ...request, unresponsiveAt: now }],
297
+ state: "unresponsive",
298
+ },
299
+ {
300
+ name: "a Stop that ended the run",
301
+ runs: [running],
302
+ turnItems: [request, signal("run_interrupt_result", running.id)],
303
+ state: null,
304
+ },
305
+ {
306
+ name: "a settled run's background work still stopping",
307
+ runs: [{ ...running, status: "completed" as const, completedAt: now }],
308
+ turnItems: [request],
309
+ state: null,
310
+ },
311
+ {
312
+ name: "a settled run whose background work ignored Stop",
313
+ runs: [{ ...running, status: "completed" as const, completedAt: now }],
314
+ turnItems: [{ ...request, unresponsiveAt: now }],
315
+ state: "unresponsive",
316
+ },
317
+ {
318
+ name: "a settled run whose background work ignored Stop, with a queued follow-up",
319
+ runs: [
320
+ { ...running, status: "completed" as const, completedAt: now },
321
+ run("run-queued", 2, "queued"),
322
+ ],
323
+ turnItems: [{ ...request, unresponsiveAt: now }],
324
+ state: "unresponsive",
325
+ },
326
+ ])("presents $name", ({ runs, turnItems, state }) => {
327
+ expect(presentRunStop({ runs, turnItems })?.state ?? null).toBe(state);
328
+ });
329
+});
330
+
331
describe("deriveRunlessWorkStartedAt", () => {
332
const later = DateTime.makeUnsafe("2026-07-28T10:05:00.000Z");
333
const rootTurn = (
packages/client-runtime/src/state/threadExecution.ts
+49
@@ -294,6 +294,55 @@ const BACKGROUND_WORK_KINDS: Record<
294
background_task: { order: 3, singular: "background task", plural: "background tasks" },
295
};
296
297
+export interface RunStopPresentation {
298
+ /** `unresponsive`: the provider did not confirm the stop in time, so Force stop is offered. */
299
+ readonly state: "stopping" | "unresponsive";
300
+ readonly runId: ThreadStructure["runs"][number]["id"];
301
+ readonly requestedAt: string;
302
+ /** The run already ended; the Stop targeted its background work. */
303
+ readonly background: boolean;
304
+}
305
+
306
+/**
307
+ * A Stop the provider has not finished yet. It stays `stopping` while the agent winds down,
308
+ * since some providers confirm the request before the turn actually ends. On a run that
309
+ * already ended, only an unanswered Stop of its background work counts.
310
+ */
311
+export function presentRunStop(
312
+ projection: Pick<ThreadStructure, "runs" | "turnItems">,
313
+): RunStopPresentation | null {
314
+ let active: ThreadStructure["runs"][number] | null = null;
315
+ let latest: ThreadStructure["runs"][number] | null = null;
316
+ for (const candidate of projection.runs) {
317
+ // Queued follow-ups never ran, so they hold no Stop of their own.
318
+ if (candidate.status !== "queued" && candidate.ordinal > (latest?.ordinal ?? -1))
319
+ latest = candidate;
320
+ if (
321
+ INTERRUPTIBLE_RUN_STATUSES.has(candidate.status) &&
322
+ candidate.ordinal > (active?.ordinal ?? -1)
323
+ )
324
+ active = candidate;
325
+ }
326
+ const run = active ?? latest;
327
+ if (run === null) return null;
328
+ let request:
329
+ | Extract<ThreadStructure["turnItems"][number], { type: "run_interrupt_request" }>
330
+ | undefined;
331
+ for (const item of projection.turnItems) {
332
+ if (item.runId !== run.id) continue;
333
+ if (item.type === "run_interrupt_result") return null;
334
+ if (item.type === "run_interrupt_request") request = item;
335
+ }
336
+ if (request === undefined || (active === null && request.unresponsiveAt === undefined))
337
+ return null;
338
+ return {
339
+ state: request.unresponsiveAt === undefined ? "stopping" : "unresponsive",
340
+ runId: run.id,
341
+ requestedAt: DateTime.formatIso(request.startedAt ?? request.updatedAt),
342
+ background: active === null,
343
+ };
344
+}
345
+
346
export interface PendingBackgroundWorkItem {
347
readonly taskId: string;
348
readonly kind: BackgroundWorkKind;
packages/contracts/src/clientRpcPermissions.ts
+2
@@ -60,6 +60,8 @@ export const CLIENT_GUARDED_RPC_SCOPES = {
60
[WS_METHODS.scheduledTasksRunNow]: AuthOrchestrationOperateScope,
61
[WS_METHODS.scheduledTasksRotateWebhookToken]: AuthOrchestrationOperateScope,
62
[ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope,
63
+ // Force stop kills agent processes, as Stop on a terminal does.
64
+ [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope,
65
// A guest link hands thread access to someone outside, like a pairing link.
66
[WS_METHODS.guestShareCreate]: AuthAccessWriteScope,
67
[WS_METHODS.peerLinksLink]: AuthAccessWriteScope,
packages/contracts/src/environment.ts
+2
@@ -99,6 +99,8 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({
99
/** Hide the switch when absent; older servers silently drop per-project provider patches. */
100
projectProviderInstanceOverrides: Schema.optionalKey(Schema.Boolean),
101
threadSnooze: Schema.optionalKey(Schema.Boolean),
102
+ /** Serves `orchestration.forceStopThread`; hide Force stop where it is absent. */
103
+ threadForceStop: Schema.optionalKey(Schema.Boolean),
104
/** Every protocol-2 server sends this; older mobile builds still read it. */
105
environmentThemes: Schema.optionalKey(Schema.Boolean),
106
usageLimitSources: Schema.optionalKey(Schema.Boolean),
packages/contracts/src/orchestrationV2.ts
+53
@@ -1676,6 +1676,8 @@ export const OrchestrationV2TurnItem = Schema.Union([
1676
...OrchestrationV2TurnItemBaseFields,
1677
type: Schema.Literal("run_interrupt_request"),
1678
message: Schema.String,
1679
+ /** Set when the provider did not confirm the stop in time; clients then offer Force stop. */
1680
+ unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtc),
1681
}),
1682
Schema.Struct({
1683
...OrchestrationV2TurnItemBaseFields,
@@ -2500,6 +2502,7 @@ export const OrchestrationV2TurnItemJson = Schema.Union([
2502
...OrchestrationV2TurnItemJsonBaseFields,
2503
type: Schema.Literal("run_interrupt_request"),
2504
message: Schema.String,
2505
+ unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtcFromString),
2506
}),
2507
Schema.Struct({
2508
...OrchestrationV2TurnItemJsonBaseFields,
@@ -3436,12 +3439,22 @@ const OrchestrationV2InternalCommand = Schema.Union([
3439
threadId: ThreadId,
3440
requestId: RuntimeRequestId,
3441
}),
3442
+ /** The provider did not confirm a Stop of this turn in time. */
3443
+ Schema.Struct({
3444
+ type: Schema.Literal("thread.interrupt.unresponsive"),
3445
+ commandId: CommandId,
3446
+ threadId: ThreadId,
3447
+ providerThreadId: ProviderThreadId,
3448
+ providerTurnId: ProviderTurnId,
3449
+ }),
3450
Schema.Struct({
3451
type: Schema.Literal("thread.background-work.settle"),
3452
commandId: CommandId,
3453
threadId: ThreadId,
3454
providerThreadId: ProviderThreadId,
3455
providerTurnId: ProviderTurnId,
3456
+ /** Replaces the Stop's own reason on the result, as Force stop does. */
3457
+ reason: Schema.optional(Schema.String),
3458
subagent: Schema.optional(
3459
Schema.Struct({
3460
id: NodeId,
@@ -3521,6 +3534,7 @@ export const ORCHESTRATION_V2_WS_METHODS = {
3534
getThreadProjection: "orchestration.getThreadProjection",
3535
getWorkflowScript: "orchestration.getWorkflowScript",
3536
stopWorkflow: "orchestration.stopWorkflow",
3537
+ forceStopThread: "orchestration.forceStopThread",
3538
getTurnItem: "orchestration.getTurnItem",
3539
launchThread: "orchestration.launchThread",
3540
subscribeShell: "orchestration.subscribeShell",
@@ -3908,6 +3922,41 @@ export class OrchestrationV2StopWorkflowError extends Schema.TaggedError<Orchest
3922
}
3923
}
3924
3925
+export const OrchestrationV2ForceStopThreadInput = Schema.Struct({
3926
+ threadId: ThreadId,
3927
+ /** The run the user saw ignore Stop; Force stop refuses once a newer run has started. */
3928
+ runId: Schema.optionalKey(RunId),
3929
+});
3930
+export type OrchestrationV2ForceStopThreadInput = typeof OrchestrationV2ForceStopThreadInput.Type;
3931
+
3932
+export const OrchestrationV2ForceStopThreadResult = Schema.Struct({
3933
+ /**
3934
+ * `killed`: the thread's own agent processes were killed. `released`: its provider session
3935
+ * was torn down, which kills the agent's process group. `releasing`: that teardown has not
3936
+ * finished yet and continues on the server. `shared`: the agent process also works for
3937
+ * `sharedWithThreadIds`, so only this thread's turn was ended.
3938
+ */
3939
+ outcome: Schema.Literals(["killed", "released", "releasing", "shared"]),
3940
+ sharedWithThreadIds: Schema.Array(ThreadId),
3941
+});
3942
+export type OrchestrationV2ForceStopThreadResult = typeof OrchestrationV2ForceStopThreadResult.Type;
3943
+
3944
+export class OrchestrationV2ForceStopThreadError extends Schema.TaggedError<OrchestrationV2ForceStopThreadError>()(
3945
+ "OrchestrationV2ForceStopThreadError",
3946
+ {
3947
+ threadId: ThreadId,
3948
+ /** `superseded`: a newer run started since the user asked; nothing was stopped. */
3949
+ reason: Schema.optional(Schema.Literal("superseded")),
3950
+ cause: Schema.optional(Schema.Defect()),
3951
+ },
3952
+) {
3953
+ override get message(): string {
3954
+ return this.reason === "superseded"
3955
+ ? "A new turn started, so nothing was force stopped."
3956
+ : "Failed to force stop the agent.";
3957
+ }
3958
+}
3959
+
3960
export const OrchestrationV2GetWorkflowScriptInput = Schema.Struct({
3961
threadId: ThreadId,
3962
scriptPath: TrimmedNonEmptyString,
@@ -4094,6 +4143,10 @@ export const OrchestrationV2RpcSchemas = {
4143
input: OrchestrationV2StopWorkflowInput,
4144
output: Schema.Void,
4145
},
4146
+ forceStopThread: {
4147
+ input: OrchestrationV2ForceStopThreadInput,
4148
+ output: OrchestrationV2ForceStopThreadResult,
4149
+ },
4150
getWorkflowScript: {
4151
input: OrchestrationV2GetWorkflowScriptInput,
4152
output: OrchestrationV2GetWorkflowScriptResult,
packages/contracts/src/rpc.ts
+8
@@ -220,6 +220,7 @@ import {
220
ORCHESTRATION_V2_WS_METHODS,
221
OrchestrationGetWorkflowScriptError,
222
OrchestrationV2StopWorkflowError,
223
+ OrchestrationV2ForceStopThreadError,
224
OrchestrationV2DispatchCommandError,
225
OrchestrationV2GetShellSnapshotError,
226
OrchestrationV2CommandOutputError,
@@ -1770,6 +1771,12 @@ const WsOrchestrationV2StopWorkflowRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.st
1771
error: Schema.Union([OrchestrationV2StopWorkflowError, EnvironmentAuthorizationError]),
1772
});
1773
1774
+const WsOrchestrationV2ForceStopThreadRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.forceStopThread, {
1775
+ payload: OrchestrationV2RpcSchemas.forceStopThread.input,
1776
+ success: OrchestrationV2RpcSchemas.forceStopThread.output,
1777
+ error: Schema.Union([OrchestrationV2ForceStopThreadError, EnvironmentAuthorizationError]),
1778
+});
1779
+
1780
const WsOrchestrationV2GetWorkflowScriptRpc = Rpc.make(
1781
ORCHESTRATION_V2_WS_METHODS.getWorkflowScript,
1782
{
@@ -2122,6 +2129,7 @@ export const WsRpcGroup = RpcGroup.make(
2129
WsOrchestrationV2DispatchCommandRpc,
2130
WsOrchestrationV2GetWorkflowScriptRpc,
2131
WsOrchestrationV2StopWorkflowRpc,
2132
+ WsOrchestrationV2ForceStopThreadRpc,
2133
WsOrchestrationV2GetTurnItemRpc,
2134
WsOrchestrationV2GetTurnDiffRpc,
2135
WsOrchestrationV2GetFullThreadDiffRpc,
packages/provider-acp/src/server/adapter.ts
+40
-21
@@ -1324,6 +1324,8 @@ interface SnapshotMessageState {
1324
loadingIndex: number;
1325
}
1326
1327
+/** How long a Stop waits for the agent to confirm before the user is offered Force stop. */
1328
+const ACP_STOP_CONFIRM_TIMEOUT_MS = 10_000;
1329
const TOOL_UPDATE_PERSIST_EVERY = 10;
1330
1331
// Agents can resend whole calls while streaming arguments.
@@ -7989,7 +7991,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
7991
return yield* Effect.uninterruptibleMask((restore) =>
7992
Effect.gen(function* () {
7993
yield* restore(runtimeTransitionPermit.take(1));
7992
- return yield* Effect.gen(function* () {
7994
+ // Stop's caller times out on this call. The transition runs in its own fiber so
7995
+ // a stalled hard teardown keeps its state and permit without holding the caller.
7996
+ const transitionFiber = yield* Effect.gen(function* () {
7997
const transition = yield* Effect.gen(function* () {
7998
const interruptContext = yield* Ref.get(activeTurn);
7999
if (
@@ -8000,8 +8004,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8004
) {
8005
// ACP cancel leaves native children alive; close confirms session-wide Stop.
8006
pendingStopCloseTurn = interruptContext;
8003
- const closed = yield* runtime.closeSession().pipe(
8004
- Effect.timeoutOption("10 seconds"),
8007
+ // Interruptible, or neither this timeout nor the caller's could end a hung close.
8008
+ const closed = yield* restore(
8009
+ runtime
8010
+ .closeSession()
8011
+ .pipe(Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS)),
8012
+ ).pipe(
8013
Effect.mapError(
8014
(cause) =>
8015
new ProviderAdapter.ProviderAdapterProtocolError({
@@ -8013,10 +8021,11 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8021
),
8022
);
8023
if (Option.isNone(closed)) {
8016
- return yield* new ProviderAdapter.ProviderAdapterProtocolError({
8024
+ return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
8025
driver,
8018
- detail:
8019
- "The agent did not confirm session close; background work may still run",
8026
+ providerThreadId: turnInput.providerThread.id,
8027
+ providerTurnId: turnInput.providerTurnId,
8028
+ timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS,
8029
});
8030
}
8031
yield* runtimeCallbackPermit.withPermit(
@@ -8246,15 +8255,16 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8255
}
8256
}
8257
const stopped = yield* Deferred.await(context.completed).pipe(
8249
- Effect.timeoutOption("10 seconds"),
8258
+ Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS),
8259
);
8260
+ // The turn stays open: the agent may still be working, and the user
8261
+ // decides whether to Force stop it.
8262
if (Option.isNone(stopped)) {
8252
- if (!context.finalized) {
8253
- yield* finalizeTurn(context, "interrupted");
8254
- }
8255
- return yield* new ProviderAdapter.ProviderAdapterProtocolError({
8263
+ return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
8264
driver,
8257
- detail: `ACP provider turn ${turnInput.providerTurnId} did not acknowledge cancellation before the interrupt timeout`,
8265
+ providerThreadId: turnInput.providerThread.id,
8266
+ providerTurnId: turnInput.providerTurnId,
8267
+ timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS,
8268
});
8269
}
8270
});
@@ -8275,20 +8285,29 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8285
}),
8286
),
8287
);
8278
- }).pipe(Effect.ensuring(runtimeTransitionPermit.release(1)));
8288
+ }).pipe(
8289
+ Effect.ensuring(runtimeTransitionPermit.release(1)),
8290
+ Effect.forkIn(sessionScope, { startImmediately: true, uninterruptible: true }),
8291
+ );
8292
+ return yield* restore(Fiber.join(transitionFiber)).pipe(
8293
+ Effect.onInterrupt(() =>
8294
+ Fiber.interrupt(transitionFiber).pipe(Effect.forkDetach, Effect.asVoid),
8295
+ ),
8296
+ );
8297
}),
8298
);
8299
},
8300
(effect, turnInput) =>
8301
effect.pipe(
8284
- Effect.mapError(
8285
- (cause) =>
8286
- new ProviderAdapter.ProviderAdapterInterruptError({
8287
- driver,
8288
- providerThreadId: turnInput.providerThread.id,
8289
- providerTurnId: turnInput.providerTurnId,
8290
- cause,
8291
- }),
8302
+ Effect.mapError((cause) =>
8303
+ cause._tag === "ProviderAdapterInterruptTimeoutError"
8304
+ ? cause
8305
+ : new ProviderAdapter.ProviderAdapterInterruptError({
8306
+ driver,
8307
+ providerThreadId: turnInput.providerThread.id,
8308
+ providerTurnId: turnInput.providerTurnId,
8309
+ cause,
8310
+ }),
8311
),
8312
),
8313
),
packages/provider-core/src/server/ProviderAdapter.ts
+16
@@ -327,6 +327,21 @@ export class ProviderAdapterInterruptError extends Schema.TaggedError<ProviderAd
327
}
328
}
329
330
+/** The provider did not confirm a stop in time; its processes may still be running. */
331
+export class ProviderAdapterInterruptTimeoutError extends Schema.TaggedError<ProviderAdapterInterruptTimeoutError>()(
332
+ "ProviderAdapterInterruptTimeoutError",
333
+ {
334
+ driver: ProviderDriverKind,
335
+ providerThreadId: ProviderThreadId,
336
+ providerTurnId: ProviderTurnId,
337
+ timeoutMs: Schema.Number,
338
+ },
339
+) {
340
+ override get message(): string {
341
+ return `The ${this.driver} provider did not confirm stopping turn ${this.providerTurnId} within ${this.timeoutMs} ms.`;
342
+ }
343
+}
344
+
345
export class ProviderAdapterRuntimeRequestResponseError extends Schema.TaggedError<ProviderAdapterRuntimeRequestResponseError>()(
346
"ProviderAdapterRuntimeRequestResponseError",
347
{
@@ -388,6 +403,7 @@ export const ProviderAdapterV2Error = Schema.Union([
403
ProviderAdapterSteerRunUnsupportedError,
404
ProviderAdapterSteerRunError,
405
ProviderAdapterInterruptError,
406
+ ProviderAdapterInterruptTimeoutError,
407
ProviderAdapterRuntimeRequestResponseError,
408
ProviderAdapterEventStreamError,
409
ProviderAdapterProtocolError,
packages/shared/src/AgentScope.ts
+7
@@ -22,6 +22,12 @@ export interface AgentScopeShape {
22
* descendants outside its process group. Preserves systemd OOM diagnosis.
23
*/
24
readonly stop: (unit: string) => Effect.Effect<void>;
25
+ /**
26
+ * SIGKILLs every process in the thread's own scope, which works even while
27
+ * the kernel throttles them. Shared scopes serve other threads and are never
28
+ * killed here. Succeeds with whether a scope was killed.
29
+ */
30
+ readonly kill: (threadId: string) => Effect.Effect<boolean>;
31
/**
32
* Forgets the thread's scope. The session manager calls it before it opens
33
* a provider session, so a scope only answers for the session that launched
@@ -42,6 +48,7 @@ export const AgentScope = Context.Reference<AgentScopeShape>("@t3tools/shared/Ag
48
wrap: ({ command, args }) => Effect.succeed({ command, args }),
49
oomKilled: () => Effect.succeed(false),
50
stop: () => Effect.void,
51
+ kill: () => Effect.succeed(false),
52
clear: () => Effect.void,
53
}),
54
});