@devries-cameron20 / t3code-1 / commits / 6b1245cced
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): Stop always completes, and an agent that ignores it can be force stopped (#27252)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 29, 2026 at 22:20 UTC 6b1245cceda1d504b6f16b438e16f2a2e08f3388
37 files changed +1583 -98
apps/mobile/src/features/threads/ThreadDetailScreen.tsx
+76
@@ -46,6 +46,7 @@ import type {
46 ScopedThreadRef,
47 ServerConfig as T3ServerConfig,
48 UsageLimitsReport,
49 + RunId,
50 } from "@t3tools/contracts";
51 import {
52 appendCodexArtifactTemplateUsePrompt,
@@ -156,8 +157,10 @@ import {
157 import { appAtomRegistry } from "../../state/atom-registry";
158 import { threadStreamHaptics } from "./threadStreamHaptics";
159 import { useAtomCommand } from "../../state/use-atom-command";
160 +import { orchestrationEnvironment } from "../../state/orchestration";
161 import { useDelayedStatus } from "../../lib/useDelayedStatus";
162 import type { PendingUserInputDraftAnswer } from "@t3tools/client-runtime/state/pending-user-input";
163 +import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution";
164 import type {
165 PendingApproval,
166 PendingUserInput,
@@ -339,6 +342,9 @@ const threadVisitTracker = createThreadVisitTracker({
342
343 // Past the composer's 180 ms height animation.
344 const COMPOSER_HEIGHT_SETTLE_MS = 220;
345 +const NO_RUN_STOP_ATOM = Atom.make<RunStopPresentation | null>(null).pipe(
346 + Atom.withLabel("mobile-thread-run-stop:empty"),
347 +);
348
349 // Anchors, the submitted message, follow state and pending scrolls belong to
350 // one thread, so the body remounts when the thread changes.
@@ -528,6 +534,50 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) {
534 const unsettleThread = useAtomCommand(threadEnvironment.unsettle, { reportFailure: false });
535 const updateMetadata = useAtomCommand(threadEnvironment.updateMetadata, { reportFailure: false });
536 const visitThread = useAtomCommand(threadEnvironment.visit, { reportFailure: false });
537 + const anyRunStop = useAtomValue(
538 + props.selectedThreadDetailRef === null
539 + ? NO_RUN_STOP_ATOM
540 + : environmentThreadDetails.runStopAtom(props.selectedThreadDetailRef),
541 + );
542 + // A background Stop matters only while that work is still pending.
543 + const runStop =
544 + anyRunStop?.background && props.selectedThread.pendingBackgroundTasks.length === 0
545 + ? null
546 + : anyRunStop;
547 + const forceStopThread = useAtomCommand(orchestrationEnvironment.forceStopThread, {
548 + reportFailure: false,
549 + });
550 + const canForceStop =
551 + useAtomValue(orchestrationEnvironment.forceStopThread.permissionAtom(props.environmentId)) &&
552 + props.serverConfig?.environment.capabilities.threadForceStop === true;
553 + const [forceStoppingKey, setForceStoppingKey] = useState<string | null>(null);
554 + // The run that ignored Stop, so a confirmation left open cannot hit a newer run.
555 + const forceStop = async (runId: RunId | undefined) => {
556 + const threadKey = selectedThreadKey;
557 + setForceStoppingKey(threadKey);
558 + const result = await forceStopThread({
559 + environmentId: props.environmentId,
560 + input:
561 + runId === undefined
562 + ? { threadId: props.selectedThread.id }
563 + : { threadId: props.selectedThread.id, runId },
564 + });
565 + setForceStoppingKey((current) => (current === threadKey ? null : current));
566 + if (result._tag === "Failure") {
567 + if (!isAtomCommandInterrupted(result)) Dialog.alert("Could not force stop the agent");
568 + return;
569 + }
570 + const others = result.value.sharedWithThreadIds.length;
571 + if (result.value.outcome === "releasing") {
572 + Dialog.alert("Turn stopped; the agent is still shutting down");
573 + }
574 + if (result.value.outcome === "shared") {
575 + Dialog.alert(
576 + "Turn stopped; the agent process kept running",
577 + `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`,
578 + );
579 + }
580 + };
581 const [statusRequest, setStatusRequest] = useState<{ threadKey: string } | null>(null);
582 const statusPending = statusRequest?.threadKey === selectedThreadKey;
583 const [lifecycleNow, setLifecycleNow] = useState(() => new Date().toISOString());
@@ -619,6 +669,32 @@ function ThreadDetailBody(props: ThreadDetailScreenProps) {
669 if (connectionStatus !== null && !props.draftMode) {
670 return connectionStatus;
671 }
672 + // Before the request check: an agent stuck on an approval may also ignore Stop.
673 + if (runStop !== null && contentPresentationKind === "ready") {
674 + const unresponsive = runStop.state === "unresponsive";
675 + return {
676 + kind: "stop",
677 + unresponsive,
678 + label: unresponsive ? "Not responding to Stop" : "Stopping…",
679 + action:
680 + unresponsive && canForceStop
681 + ? {
682 + label: forceStoppingKey === selectedThreadKey ? "Force stopping..." : "Force stop",
683 + accessibilityLabel: "Force stop the agent",
684 + disabled: forceStoppingKey === selectedThreadKey,
685 + onPress: () =>
686 + Dialog.confirm({
687 + title: "Force stop the agent?",
688 + message:
689 + "It has not answered Stop. Force stop ends the turn and kills the agent's processes, unless other working threads share them.",
690 + confirmText: "Force stop",
691 + destructive: true,
692 + onConfirm: () => void forceStop(runStop.runId),
693 + }),
694 + }
695 + : null,
696 + };
697 + }
698 if (props.activePendingApproval !== null || props.activePendingUserInput !== null) {
699 return null;
700 }
apps/mobile/src/features/threads/floating-working-control.tsx
+27 -1
@@ -83,7 +83,9 @@ export function FloatingWorkingControl(props: {
83 const [queueWidth, setQueueWidth] = useState(0);
84 const [actionWidth, setActionWidth] = useState(0);
85 const action =
86 - props.status?.kind === "background" || props.status?.kind === "lifecycle"
86 + props.status?.kind === "background" ||
87 + props.status?.kind === "lifecycle" ||
88 + props.status?.kind === "stop"
89 ? props.status.action
90 : null;
91 const [agentsWidth, setAgentsWidth] = useState(0);
@@ -471,6 +473,30 @@ function FloatingStatusLabel(props: {
473 </StatusLabelRow>
474 );
475 }
476 + if (props.status.kind === "stop") {
477 + return (
478 + <StatusLabelRow
479 + key="stop"
480 + accessibilityLabel={props.status.label}
481 + className="gap-2"
482 + onLayout={props.onLayout}
483 + >
484 + {props.status.unresponsive ? (
485 + <SymbolView
486 + name="exclamationmark.triangle"
487 + size={13}
488 + tintColorClassName="accent-warning-foreground"
489 + type="monochrome"
490 + />
491 + ) : (
492 + <AppActivityIndicator size="small" colorClassName="accent-icon-muted" />
493 + )}
494 + <Text className="shrink font-t3-medium text-xs text-foreground" numberOfLines={1}>
495 + {props.status.label}
496 + </Text>
497 + </StatusLabelRow>
498 + );
499 + }
500 if (props.status.kind === "goal") {
501 return (
502 <StatusLabelRow
apps/mobile/src/features/threads/floating-working-status.ts
+6
@@ -31,6 +31,12 @@ export type FloatingWorkingStatus =
31 readonly label: string;
32 readonly action: FloatingStatusAction;
33 }
34 + | {
35 + readonly kind: "stop";
36 + readonly label: string;
37 + readonly unresponsive: boolean;
38 + readonly action: FloatingStatusAction | null;
39 + }
40 | { readonly kind: "goal"; readonly label: string; readonly accessibilityLabel: string }
41 | { readonly kind: "preparing"; readonly label: string }
42 | {
apps/server/src/auth/RpcAuthorization.ts
+1
@@ -42,6 +42,7 @@ export const RPC_REQUIRED_SCOPES = {
42 [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: AuthOrchestrationOperateScope,
43 [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: AuthOrchestrationReadScope,
44 [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope,
45 + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope,
46 [ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: AuthOrchestrationReadScope,
47 [ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: AuthOrchestrationReadScope,
48 [ORCHESTRATION_V2_WS_METHODS.searchThreads]: AuthOrchestrationReadScope,
apps/server/src/environment/ServerEnvironment.ts
+1
@@ -240,6 +240,7 @@ export const make = Effect.gen(function* () {
240 projectSettingsOverrides: true,
241 projectProviderInstanceOverrides: true,
242 threadSnooze: true,
243 + threadForceStop: true,
244 environmentThemes: true,
245 usageLimitSources: true,
246 providerWorkspaceSnapshotsOnDemand: true,
apps/server/src/observability/RpcInstrumentation.ts
+1
@@ -14,6 +14,7 @@ const RPC_AGGREGATES = {
14 [ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: "orchestrationV2",
15 [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: "orchestration",
16 [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: "orchestration",
17 + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: "orchestration",
18 [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: "orchestration",
19 [ORCHESTRATION_V2_WS_METHODS.getTurnDiff]: "orchestration",
20 [ORCHESTRATION_V2_WS_METHODS.getFullThreadDiff]: "orchestration",
apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts
+90
@@ -7498,6 +7498,96 @@ describe("AcpAdapterV2", () => {
7498 }).pipe(Effect.provide(layerTest), Effect.scoped),
7499 );
7500
7501 + it.live("lets a caller abandon Stop while hard teardown stalls, and finishes the teardown", () =>
7502 + Effect.gen(function* () {
7503 + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
7504 + const idAllocator = yield* IdAllocator.IdAllocatorV2;
7505 + const path = yield* Path.Path;
7506 + const selfInvocation = yield* resolveSelfInvocation();
7507 + const mockAgentPath = yield* path.fromFileUrl(
7508 + new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
7509 + );
7510 + const protocolEvents = yield* Queue.bounded<EffectAcpProtocol.AcpProtocolLogEvent>(256);
7511 + const teardownStarted = yield* Deferred.make<void>();
7512 + const releaseTeardown = yield* Deferred.make<void>();
7513 + let runtimeOrdinalSeen = 0;
7514 + const instanceId = ProviderInstanceId.make("acp-test");
7515 + const adapter = yield* makeAcpAdapterV2({
7516 + instanceId,
7517 + flavor: {
7518 + driver: ACP_TEST_DRIVER,
7519 + capabilities: AcpProviderCapabilitiesV2,
7520 + interruptPromptOnCancel: true,
7521 + restartRuntimeAfterInterrupt: true,
7522 + terminateRuntimeProcessGroupOnInterrupt: true,
7523 + makeRuntime: makeMockRuntime({
7524 + childProcessSpawner,
7525 + mockAgentPath,
7526 + ownDetachedProcessGroup: true,
7527 + processGroupPlatform: "win32",
7528 + windowsProcessTreeTerminator: () =>
7529 + Deferred.succeed(teardownStarted, undefined).pipe(
7530 + Effect.andThen(Deferred.await(releaseTeardown)),
7531 + ),
7532 + environment: (runtimeOrdinal) => {
7533 + runtimeOrdinalSeen = runtimeOrdinal;
7534 + return runtimeOrdinal === 1 ? { T3_ACP_HANG_PROMPT_FOREVER: "1" } : {};
7535 + },
7536 + protocolEvents,
7537 + }),
7538 + },
7539 + selfInvocation,
7540 + });
7541 + const threadId = ThreadId.make("thread-acp-abandoned-hard-teardown");
7542 + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({
7543 + runtimeMode: "full-access",
7544 + interactionMode: "default",
7545 + cwd: process.cwd(),
7546 + });
7547 + const modelSelection = { instanceId, model: "default" } as const;
7548 + const runtime = yield* adapter.openSession({
7549 + threadId,
7550 + providerSessionId: ProviderSessionId.make("provider-session-acp-abandoned-hard-teardown"),
7551 + modelSelection,
7552 + runtimePolicy,
7553 + });
7554 + const providerThread = yield* runtime.ensureThread({
7555 + threadId,
7556 + modelSelection,
7557 + runtimePolicy,
7558 + });
7559 + const now = yield* DateTime.now;
7560 + yield* runtime
7561 + .startTurn(
7562 + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 1 }),
7563 + )
7564 + .pipe(Effect.forkScoped);
7565 + const providerTurnId = idAllocator.derive.providerTurn({
7566 + driver: ACP_TEST_DRIVER,
7567 + nativeTurnId: acpScopedNativeId(instanceId, "mock-session-1:turn:1"),
7568 + });
7569 + yield* pollProtocolMethods(protocolEvents);
7570 + const interruptFiber = yield* runtime
7571 + .interruptTurn({ providerThread, providerTurnId, requestRuntimeRestart: true })
7572 + .pipe(Effect.forkScoped);
7573 + yield* Deferred.await(teardownStarted);
7574 +
7575 + // The session manager's Stop timeout interrupts this call; it must not wait for teardown.
7576 + yield* Fiber.interrupt(interruptFiber);
7577 +
7578 + const startFiber = yield* runtime
7579 + .startTurn(
7580 + makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now, ordinal: 2 }),
7581 + )
7582 + .pipe(Effect.forkScoped);
7583 + yield* Deferred.succeed(releaseTeardown, undefined);
7584 + yield* Fiber.join(startFiber);
7585 + assert.equal(runtimeOrdinalSeen, 2);
7586 + const replacementMethods = yield* pollProtocolMethods(protocolEvents);
7587 + assert.equal(replacementMethods.filter((method) => method === "session/resume").length, 1);
7588 + }).pipe(Effect.provide(layerTest), Effect.scoped),
7589 + );
7590 +
7591 it.effect(
7592 "direct Stop on a deferred subagent hold terminalizes the subagent and does not carry it forward",
7593 () =>
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+20 -26
@@ -322,6 +322,8 @@ const WORKFLOW_MEMBER_TRANSCRIPT_FLUSH_MS = 500;
322 // A steer waits at most this long, from its request, for its foreground
323 // subagents to move to the background before it is sent.
324 const BACKGROUND_MOVE_TIMEOUT_MS = 5_000;
325 +// How long a Stop waits for the CLI to exit before the user is offered Force stop.
326 +const CLAUDE_STOP_CONFIRM_TIMEOUT_MS = 10_000;
327
328 const CLAUDE_GROUP_WATCH_SCRIPT = [
329 "{ trap '' TERM; read -r _ <&3; kill -TERM 0; sleep 5; kill -KILL 0; } </dev/null >/dev/null 2>&1 &",
@@ -10471,44 +10473,36 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10473 yield* existing.query.interrupt;
10474 yield* existing.query.close.pipe(Effect.ignore);
10475 const closed = yield* Deferred.await(existing.closed).pipe(
10474 - Effect.timeoutOption("10 seconds"),
10476 + Effect.timeoutOption(CLAUDE_STOP_CONFIRM_TIMEOUT_MS),
10477 );
10478 if (Option.isSome(closed)) {
10479 return;
10480 }
10479 -
10480 - const completedAt = yield* DateTime.now;
10481 + // The turn stays open: the CLI may still be working, and the user decides
10482 + // whether to Force stop it.
10483 yield* Effect.logWarning("orchestration-v2.claude-query-interrupt-timeout", {
10484 providerSessionId: input.providerSessionId,
10485 providerThreadId: turnInput.providerThread.id,
10486 providerTurnId: turnInput.providerTurnId,
10487 });
10486 - const ownedLiveQuery = yield* Ref.modify(queryContext, (current) =>
10487 - current?.query === existing.query ? [true, null] : [false, current],
10488 - );
10489 - if (ownedLiveQuery) {
10490 - existing.stopping = true;
10491 - yield* finalizeActiveTurnAfterQueryExit(existing);
10492 - yield* endSubagentCallsOfClosedQuery(existing.nativeThreadId, "interrupted");
10493 - } else {
10494 - yield* finalizeActiveTurn({
10495 - context: currentTurn,
10496 - status: "interrupted",
10497 - completedAt,
10498 - });
10499 - }
10500 - yield* Deferred.succeed(existing.closed, undefined);
10488 + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
10489 + driver: CLAUDE_PROVIDER,
10490 + providerThreadId: turnInput.providerThread.id,
10491 + providerTurnId: turnInput.providerTurnId,
10492 + timeoutMs: CLAUDE_STOP_CONFIRM_TIMEOUT_MS,
10493 + });
10494 },
10495 (effect, turnInput) =>
10496 effect.pipe(
10504 - Effect.mapError(
10505 - (cause) =>
10506 - new ProviderAdapter.ProviderAdapterInterruptError({
10507 - driver: CLAUDE_PROVIDER,
10508 - providerThreadId: turnInput.providerThread.id,
10509 - providerTurnId: turnInput.providerTurnId,
10510 - cause,
10511 - }),
10497 + Effect.mapError((cause) =>
10498 + cause._tag === "ProviderAdapterInterruptTimeoutError"
10499 + ? cause
10500 + : new ProviderAdapter.ProviderAdapterInterruptError({
10501 + driver: CLAUDE_PROVIDER,
10502 + providerThreadId: turnInput.providerThread.id,
10503 + providerTurnId: turnInput.providerTurnId,
10504 + cause,
10505 + }),
10506 ),
10507 ),
10508 );
apps/server/src/orchestration-v2/EffectWorker.test.ts
+17 -3
@@ -111,6 +111,8 @@ function layerExecutorFor(input: {
111 closeInstance: () => Effect.void,
112 release: () => record("release"),
113 detach: () => record("detach"),
114 + otherWorkingThreads: () => Effect.succeed([]),
115 + releaseUnlessBusy: () => Effect.succeed(true),
116 }),
117 ),
118 Layer.succeed(
@@ -167,7 +169,18 @@ function layerExecutorFor(input: {
169 );
170 }
171
170 -it.effect("settles a stopped run when its adapter has already lost the native turn", () =>
172 +it.effect.each([
173 + {
174 + name: "settles a stopped run when its adapter has already lost the native turn",
175 + unresponsive: false,
176 + dispatched: ["thread.background-work.settle"],
177 + },
178 + {
179 + name: "leaves the run running and flags the Stop when the provider never confirms it",
180 + unresponsive: true,
181 + dispatched: ["thread.interrupt.unresponsive"],
182 + },
183 +])("$name", ({ unresponsive, dispatched }) =>
184 Effect.gen(function* () {
185 const now = yield* DateTime.now;
186 const events = yield* Ref.make<ReadonlyArray<string>>([]);
@@ -178,7 +191,8 @@ it.effect("settles a stopped run when its adapter has already lost the native tu
191 threadId,
192 operation: "interrupt",
193 providerTurnId,
181 - cause: "Provider turn is not active.",
194 + ...(unresponsive ? { unresponsive } : {}),
195 + cause: unresponsive ? "Timed out." : "Provider turn is not active.",
196 }),
197 threads: {
198 dispatch: (command) =>
@@ -199,7 +213,7 @@ it.effect("settles a stopped run when its adapter has already lost the native tu
213 },
214 });
215 }).pipe(Effect.provide(layer));
202 - assert.deepEqual(yield* Ref.get(events), ["thread.background-work.settle"]);
216 + assert.deepEqual(yield* Ref.get(events), dispatched);
217 }),
218 );
219
apps/server/src/orchestration-v2/EffectWorker.ts
+32 -17
@@ -177,6 +177,13 @@ export const layerExecutor: Layer.Layer<
177 cause,
178 }),
179 ),
180 + Effect.map((outcome) => ({ unresponsive: false, outcome }) as const),
181 + // Settling here would end the run while the agent may still be running; the
182 + // user decides with Force stop instead.
183 + Effect.catchIf(
184 + (cause) => request.subagent === undefined && cause.unresponsive === true,
185 + () => Effect.succeed({ unresponsive: true, outcome: undefined } as const),
186 + ),
187 Effect.catchIf(
188 (cause) =>
189 request.subagent === undefined &&
@@ -184,27 +191,35 @@ export const layerExecutor: Layer.Layer<
191 request.type,
192 Cause.pretty(Cause.fail(cause)),
193 ),
187 - () => Effect.void,
194 + () => Effect.succeed({ unresponsive: false, outcome: undefined } as const),
195 ),
189 - Effect.andThen((outcome) =>
190 - request.subagent !== undefined && outcome === undefined
191 - ? Effect.void
192 - : threads.dispatch({
193 - type: "thread.background-work.settle",
194 - commandId: CommandId.make(`${effect.id}:background-work-settled`),
195 - ...(request.subagent === undefined || outcome === undefined
196 - ? {}
197 - : {
198 - subagent: {
199 - id: request.subagent.id,
200 - stoppedAt: DateTime.makeUnsafe(effect.createdAt),
201 - outcome,
202 - },
203 - }),
196 + Effect.andThen(({ unresponsive, outcome }) =>
197 + unresponsive
198 + ? threads.dispatch({
199 + type: "thread.interrupt.unresponsive",
200 + commandId: CommandId.make(`${effect.id}:unresponsive`),
201 threadId: effect.threadId,
202 providerThreadId: request.providerThreadId,
203 providerTurnId: request.providerTurnId,
207 - }),
204 + })
205 + : request.subagent !== undefined && outcome === undefined
206 + ? Effect.void
207 + : threads.dispatch({
208 + type: "thread.background-work.settle",
209 + commandId: CommandId.make(`${effect.id}:background-work-settled`),
210 + ...(request.subagent === undefined || outcome === undefined
211 + ? {}
212 + : {
213 + subagent: {
214 + id: request.subagent.id,
215 + stoppedAt: DateTime.makeUnsafe(effect.createdAt),
216 + outcome,
217 + },
218 + }),
219 + threadId: effect.threadId,
220 + providerThreadId: request.providerThreadId,
221 + providerTurnId: request.providerTurnId,
222 + }),
223 ),
224 Effect.mapError(
225 (cause) =>
apps/server/src/orchestration-v2/Orchestrator.ts
+58 -1
@@ -536,6 +536,7 @@ function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId {
536 case "checkpoint.rollback":
537 case "checkpoint.rollback.fail":
538 case "thread.background-work.settle":
539 + case "thread.interrupt.unresponsive":
540 case "thread.stop":
541 case "provider.switch":
542 return command.threadId;
@@ -10278,7 +10279,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10279 run,
10280 rootNode,
10281 providerThread,
10281 - reason: yield* runInterruptReason(input.command, input.events, run.id),
10282 + reason:
10283 + (input.command.type === "thread.background-work.settle"
10284 + ? input.command.reason
10285 + : undefined) ?? (yield* runInterruptReason(input.command, input.events, run.id)),
10286 completedAt: input.now,
10287 }),
10288 });
@@ -10525,6 +10529,55 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10529 yield* closePendingLinkRequests({ ...input, threadId: thread.id });
10530 });
10531
10532 + const dispatchInterruptUnresponsive = Effect.fn("Orchestrator.dispatchInterruptUnresponsive")(
10533 + function* (
10534 + command: Extract<
10535 + OrchestrationV2InternalCommand,
10536 + { readonly type: "thread.interrupt.unresponsive" }
10537 + >,
10538 + events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
10539 + ) {
10540 + const { providerTurn } = yield* projectionStore
10541 + .getProviderControlContext(command.threadId, {
10542 + providerThreadId: command.providerThreadId,
10543 + providerTurnId: command.providerTurnId,
10544 + })
10545 + .pipe(mapDispatchError(command));
10546 + const { runs } = yield* projectionStore
10547 + .getThreadRecords(command.threadId, ["runs"])
10548 + .pipe(mapDispatchError(command));
10549 + const run = runs.find(
10550 + (candidate) =>
10551 + providerTurn !== undefined && candidate.activeAttemptId === providerTurn.runAttemptId,
10552 + );
10553 + // A settled run still counts: Stop may have targeted its background work.
10554 + if (run === undefined) return;
10555 + const { turnItems } = yield* projectionStore
10556 + .getThreadRecords(command.threadId, ["turnItems"], {
10557 + turnItemTypes: ["run_interrupt_request", "run_interrupt_result"],
10558 + turnItemRunIds: [run.id],
10559 + })
10560 + .pipe(mapDispatchError(command));
10561 + // A provider that stopped late already wrote the result; it needs no Force stop.
10562 + if (turnItems.some((item) => item.type === "run_interrupt_result")) return;
10563 + const request = turnItems.find((item) => item.type === "run_interrupt_request");
10564 + if (request?.type !== "run_interrupt_request" || request.unresponsiveAt !== undefined) return;
10565 + const now = yield* DateTime.now;
10566 + yield* emit(
10567 + events,
10568 + command,
10569 + )({
10570 + type: "turn-item.updated",
10571 + threadId: command.threadId,
10572 + runId: run.id,
10573 + ...(request.nodeId === null ? {} : { nodeId: request.nodeId }),
10574 + providerInstanceId: run.providerInstanceId,
10575 + occurredAt: now,
10576 + payload: { ...request, unresponsiveAt: now, updatedAt: now },
10577 + });
10578 + },
10579 + );
10580 +
10581 const dispatchBackgroundWorkSettle = Effect.fn("Orchestrator.dispatchBackgroundWorkSettle")(
10582 function* (
10583 command: Extract<
@@ -12902,6 +12955,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12955 case "thread.background-work.settle":
12956 yield* dispatchBackgroundWorkSettle(command, events, effects);
12957 break;
12958 + case "thread.interrupt.unresponsive":
12959 + yield* dispatchInterruptUnresponsive(command, events);
12960 + break;
12961 case "thread.fork":
12962 yield* dispatchThreadFork(command, events);
12963 break;
@@ -13032,6 +13088,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13088 (planned) =>
13089 planned.events.length > 0 ||
13090 command.type === "thread.background-work.settle" ||
13091 + command.type === "thread.interrupt.unresponsive" ||
13092 command.type === "thread.stop" ||
13093 command.type === "subagent.stop" ||
13094 // Closing a share again, or after it ended, is not an error.
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+157 -1
@@ -338,6 +338,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
338 readonly backgroundTasksEndWithProcess?: boolean;
339 readonly hangSessionScopeClose?: boolean;
340 readonly startTurn?: Effect.Effect<void>;
341 + readonly interruptTurn?: Effect.Effect<void>;
342 readonly beforeUnload?: Effect.Effect<void>;
343 readonly spawnBeforeOpen?: boolean;
344 readonly scopeCloseReached?: Deferred.Deferred<void>;
@@ -437,7 +438,7 @@ const makeProviderAdapter = Effect.fnUntraced(function* (
438 Ref.update(state, (current) => ({
439 ...current,
440 interruptCount: current.interruptCount + 1,
440 - })),
441 + })).pipe(Effect.andThen(options.interruptTurn ?? Effect.void)),
442 unloadThread: ({ providerThread }) =>
443 (options.beforeUnload ?? Effect.void).pipe(
444 Effect.andThen(
@@ -485,6 +486,7 @@ function layerTest(input: {
486 readonly backgroundTasksEndWithProcess?: boolean;
487 readonly hangSessionScopeClose?: boolean;
488 readonly startTurn?: Effect.Effect<void>;
489 + readonly interruptTurn?: Effect.Effect<void>;
490 readonly beforeUnload?: Effect.Effect<void>;
491 readonly spawnBeforeOpen?: boolean;
492 readonly scopeCloseReached?: Deferred.Deferred<void>;
@@ -520,6 +522,7 @@ function layerTest(input: {
522 ? {}
523 : { hangSessionScopeClose: input.hangSessionScopeClose }),
524 ...(input.startTurn === undefined ? {} : { startTurn: input.startTurn }),
525 + ...(input.interruptTurn === undefined ? {} : { interruptTurn: input.interruptTurn }),
526 ...(input.beforeUnload === undefined ? {} : { beforeUnload: input.beforeUnload }),
527 ...(input.spawnBeforeOpen === undefined ? {} : { spawnBeforeOpen: input.spawnBeforeOpen }),
528 ...(input.scopeCloseReached === undefined
@@ -1367,6 +1370,159 @@ it.effect(
1370 }),
1371 );
1372
1373 +it.effect("ProviderSessionManagerV2 gives up on a provider that never confirms a Stop", () =>
1374 + Effect.gen(function* () {
1375 + const state = yield* Ref.make(emptyState);
1376 + yield* Effect.gen(function* () {
1377 + const eventSink = yield* EventSink.EventSinkV2;
1378 + const idAllocator = yield* IdAllocator.IdAllocatorV2;
1379 + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1380 + const now = yield* DateTime.now;
1381 + const threadId = ThreadId.make("thread-provider-session-manager-unresponsive-stop");
1382 + const providerSessionId = yield* idAllocator.allocate.providerSession({
1383 + providerInstanceId: modelSelection.instanceId,
1384 + threadId,
1385 + });
1386 + yield* eventSink.write({
1387 + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
1388 + });
1389 + const runtime = yield* manager.open({
1390 + threadId,
1391 + providerSessionId,
1392 + modelSelection,
1393 + runtimePolicy,
1394 + });
1395 + const stopping = yield* runtime
1396 + .interruptTurn({
1397 + providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }),
1398 + providerTurnId: idAllocator.derive.providerTurn({
1399 + driver: CODEX_DRIVER,
1400 + nativeTurnId: "native-turn-unresponsive",
1401 + }),
1402 + })
1403 + .pipe(Effect.flip, Effect.forkChild);
1404 + yield* TestClock.adjust(ProviderSessionManager.PROVIDER_INTERRUPT_TIMEOUT_MS);
1405 + assert.equal((yield* Fiber.join(stopping))._tag, "ProviderAdapterInterruptTimeoutError");
1406 + }).pipe(
1407 + Effect.provide(layerTest({ state, idleTimeoutMs: 60_000, interruptTurn: Effect.never })),
1408 + );
1409 + }),
1410 +);
1411 +
1412 +it.effect(
1413 + "ProviderSessionManagerV2 keeps a shared session a sibling is busy on when releasing",
1414 + () =>
1415 + Effect.gen(function* () {
1416 + const state = yield* Ref.make(emptyState);
1417 + yield* Effect.gen(function* () {
1418 + const eventSink = yield* EventSink.EventSinkV2;
1419 + const idAllocator = yield* IdAllocator.IdAllocatorV2;
1420 + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1421 + const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1422 + const now = yield* DateTime.now;
1423 + const stopping = ThreadId.make("thread-provider-session-manager-release-stopping");
1424 + const sibling = ThreadId.make("thread-provider-session-manager-release-sibling");
1425 + const providerSessionId = idAllocator.derive.providerSession({
1426 + providerInstanceId: modelSelection.instanceId,
1427 + });
1428 + yield* eventSink.write({
1429 + events: [
1430 + yield* makeThreadCreatedEvent({ idAllocator, threadId: stopping, now }),
1431 + yield* makeThreadCreatedEvent({ idAllocator, threadId: sibling, now }),
1432 + ],
1433 + });
1434 + const runtime = yield* manager.open({
1435 + threadId: stopping,
1436 + providerSessionId,
1437 + modelSelection,
1438 + runtimePolicy,
1439 + });
1440 + yield* manager.open({
1441 + threadId: sibling,
1442 + providerSessionId,
1443 + modelSelection,
1444 + runtimePolicy,
1445 + });
1446 + const providerThreadOf = (threadId: ThreadId) =>
1447 + makeProviderThread({
1448 + idAllocator,
1449 + threadId,
1450 + providerSessionId,
1451 + now,
1452 + nativeThreadId: `native-${threadId}`,
1453 + });
1454 + const runId = idAllocator.derive.run({ threadId: sibling, ordinal: 1 });
1455 + yield* runtime.startTurn({
1456 + appThread: (yield* projectionStore.getThreadProjection(sibling)).thread,
1457 + threadId: sibling,
1458 + runId,
1459 + runOrdinal: 1,
1460 + providerTurnOrdinal: 1,
1461 + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }),
1462 + rootNodeId: idAllocator.derive.rootNode({ runId }),
1463 + providerThread: providerThreadOf(sibling),
1464 + message: {
1465 + createdBy: "user",
1466 + creationSource: "web",
1467 + messageId: yield* idAllocator.allocate.message({ threadId: sibling, ordinal: 1 }),
1468 + text: "turn",
1469 + attachments: [],
1470 + },
1471 + modelSelection,
1472 + runtimePolicy,
1473 + });
1474 + const release = (own: ThreadId) =>
1475 + manager.releaseUnlessBusy({
1476 + providerSessionId,
1477 + ownProviderThreadIds: [providerThreadOf(own).id],
1478 + reason: "manual_shutdown",
1479 + });
1480 +
1481 + assert.isFalse(yield* release(stopping));
1482 + assert.isTrue(yield* manager.isLive(providerSessionId));
1483 + assert.isTrue(yield* release(sibling));
1484 + assert.isFalse(yield* manager.isLive(providerSessionId));
1485 +
1486 + // A turn that starts on a runtime released meanwhile fails instead of running on it.
1487 + const startOnReleasedRuntime = Effect.gen(function* () {
1488 + const lateRunId = idAllocator.derive.run({ threadId: stopping, ordinal: 1 });
1489 + return yield* Effect.flip(
1490 + runtime.startTurn({
1491 + appThread: (yield* projectionStore.getThreadProjection(stopping)).thread,
1492 + threadId: stopping,
1493 + runId: lateRunId,
1494 + runOrdinal: 1,
1495 + providerTurnOrdinal: 1,
1496 + attemptId: idAllocator.derive.runAttempt({ runId: lateRunId, attemptOrdinal: 1 }),
1497 + rootNodeId: idAllocator.derive.rootNode({ runId: lateRunId }),
1498 + providerThread: providerThreadOf(stopping),
1499 + message: {
1500 + createdBy: "user",
1501 + creationSource: "web",
1502 + messageId: yield* idAllocator.allocate.message({ threadId: stopping, ordinal: 1 }),
1503 + text: "turn",
1504 + attachments: [],
1505 + },
1506 + modelSelection,
1507 + runtimePolicy,
1508 + }),
1509 + );
1510 + });
1511 + assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError");
1512 +
1513 + // Nor may it leave its turn busy on a replacement opened under the same id.
1514 + yield* manager.open({
1515 + threadId: sibling,
1516 + providerSessionId,
1517 + modelSelection,
1518 + runtimePolicy,
1519 + });
1520 + assert.equal((yield* startOnReleasedRuntime)._tag, "ProviderAdapterTurnStartError");
1521 + assert.isTrue(yield* release(sibling));
1522 + }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60 * 60_000 })));
1523 + }),
1524 +);
1525 +
1526 it.effect("ProviderSessionManagerV2 stops a session still opening when its layer shuts down", () =>
1527 Effect.gen(function* () {
1528 const state = yield* Ref.make(emptyState);
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+128 -11
@@ -57,11 +57,19 @@ import * as ProjectionStore from "./ProjectionStore.ts";
57 const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
58 const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000;
59 const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000;
60 +/** Past adapters' own 10 s stop fallbacks, so they settle first when the provider answers at all. */
61 +export const PROVIDER_INTERRUPT_TIMEOUT_MS = 15 * 1000;
62
63 const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThreadId}#`;
64 const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) =>
65 `${busyTurnPrefix(providerThreadId)}${runOrdinal}`;
66 const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000;
67 +const WORKING_RUN_STATUSES: ReadonlySet<string> = new Set([
68 + "preparing",
69 + "starting",
70 + "running",
71 + "waiting",
72 +]);
73
74 export const ProviderSessionReleaseReason = Schema.Literals([
75 "idle_timeout",
@@ -181,6 +189,25 @@ export interface ProviderSessionManagerV2Shape {
189 readonly detail?: string;
190 readonly revokeMcpCredential?: boolean;
191 }) => Effect.Effect<void, ProviderSessionManagerV2Error>;
192 + /**
193 + * Releases the session unless, at the moment of removal, a turn of a provider thread outside
194 + * `ownProviderThreadIds` is busy on it. Checked atomically with turn starts. Succeeds with
195 + * whether it was released.
196 + */
197 + readonly releaseUnlessBusy: (input: {
198 + readonly providerSessionId: ProviderSessionId;
199 + readonly ownProviderThreadIds: ReadonlyArray<ProviderThreadId>;
200 + readonly reason: ProviderSessionReleaseReason;
201 + readonly detail?: string;
202 + }) => Effect.Effect<boolean, ProviderSessionManagerV2Error>;
203 + /**
204 + * Other threads with a run or background work on this session's shared process; empty when
205 + * it serves one thread.
206 + */
207 + readonly otherWorkingThreads: (input: {
208 + readonly providerSessionId: ProviderSessionId;
209 + readonly threadId: ThreadId;
210 + }) => Effect.Effect<ReadonlyArray<ThreadId>>;
211 }
212
213 export class ProviderSessionManagerV2 extends Context.Service<
@@ -895,6 +922,7 @@ export const layerWithOptions = (
922 function* (input: {
923 readonly providerSessionId: ProviderSessionId;
924 readonly onlyIfIdleGeneration?: number;
925 + readonly keepIfBusyOtherThan?: ReadonlyArray<ProviderThreadId>;
926 }): Effect.fn.Return<readonly [Option.Option<LiveSessionEntry>, DateTime.Utc]> {
927 const key = sessionKey(input.providerSessionId);
928 const candidate = (yield* Ref.get(sessions)).get(key);
@@ -914,6 +942,15 @@ export const layerWithOptions = (
942 ) {
943 return ["kept", current] as const;
944 }
945 + const own = input.keepIfBusyOtherThan;
946 + if (
947 + own !== undefined &&
948 + [...existing.busyTurns].some(
949 + (turnKey) => !own.some((id) => turnKey.startsWith(busyTurnPrefix(id))),
950 + )
951 + ) {
952 + return ["kept", current] as const;
953 + }
954 const updated = new Map(current);
955 updated.delete(key);
956 return ["removed", updated] as const;
@@ -972,6 +1009,7 @@ export const layerWithOptions = (
1009 readonly detachedThreadId?: ThreadId;
1010 readonly cancelIdleFiber?: boolean;
1011 readonly onlyIfIdleGeneration?: number;
1012 + readonly keepIfBusyOtherThan?: ReadonlyArray<ProviderThreadId>;
1013 readonly gracefulSubscribers?: boolean;
1014 }) =>
1015 Effect.acquireUseRelease(
@@ -1380,15 +1418,21 @@ export const layerWithOptions = (
1418 .pipe(Effect.ensuring(Effect.sync(dropReservation)));
1419 });
1420
1383 - const markBusy = (providerSessionId: ProviderSessionId, turnKey: string) =>
1421 + // Marks only the entry this runtime belongs to: a released or replaced session must not
1422 + // gain a busy turn that never ends. Succeeds with whether the turn was marked.
1423 + const markBusy = (
1424 + providerSessionId: ProviderSessionId,
1425 + turnKey: string,
1426 + runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime,
1427 + ) =>
1428 withActivityError(
1429 providerSessionId,
1430 Effect.gen(function* () {
1431 const key = sessionKey(providerSessionId);
1388 - const idleFiber = yield* Ref.modify(sessions, (current) => {
1432 + const marked = yield* Ref.modify(sessions, (current) => {
1433 const entry = current.get(key);
1390 - if (entry === undefined) {
1391 - return [null, current] as const;
1434 + if (entry === undefined || entry.runtime !== runtime) {
1435 + return [undefined, current] as const;
1436 }
1437 const updated = new Map(current);
1438 updated.set(key, {
@@ -1397,9 +1441,11 @@ export const layerWithOptions = (
1441 idleFiber: null,
1442 pinnedSinceMs: null,
1443 });
1400 - return [entry.idleFiber, updated] as const;
1444 + return [{ idleFiber: entry.idleFiber }, updated] as const;
1445 });
1402 - yield* cancelIdleFiber(idleFiber);
1446 + if (marked === undefined) return false;
1447 + yield* cancelIdleFiber(marked.idleFiber);
1448 + return true;
1449 }),
1450 );
1451
@@ -1785,11 +1831,31 @@ export const layerWithOptions = (
1831 ),
1832 Effect.andThen(
1833 Effect.acquireUseRelease(
1788 - observeActivity(
1834 + markBusy(
1835 providerSessionId,
1790 - markBusy(
1791 - providerSessionId,
1792 - busyTurnKey(input.providerThread.id, input.runOrdinal),
1836 + busyTurnKey(input.providerThread.id, input.runOrdinal),
1837 + runtime,
1838 + ).pipe(
1839 + Effect.catchCause((cause) =>
1840 + Effect.logWarning("orchestration-v2.driver-session.activity-failed", {
1841 + providerSessionId,
1842 + cause,
1843 + }).pipe(Effect.as(true)),
1844 + ),
1845 + // A release between attach and markBusy leaves nothing to mark: fail the
1846 + // start instead of running it on a runtime being torn down.
1847 + Effect.flatMap((marked) =>
1848 + marked
1849 + ? Effect.void
1850 + : Effect.fail(
1851 + new ProviderAdapter.ProviderAdapterTurnStartError({
1852 + driver: runtime.driver,
1853 + threadId: input.threadId,
1854 + providerThreadId: input.providerThread.id,
1855 + runId: input.runId,
1856 + cause: "The provider session was released while the turn started.",
1857 + }),
1858 + ),
1859 ),
1860 ),
1861 () =>
@@ -1808,9 +1874,26 @@ export const layerWithOptions = (
1874 observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
1875 Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))),
1876 ),
1877 + // A provider starved of memory can stop answering, so no caller may wait on it unbounded.
1878 interruptTurn: (input) =>
1879 observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
1813 - Effect.andThen(runtime.interruptTurn(input).pipe(turnMetrics("interrupt"))),
1880 + Effect.andThen(
1881 + runtime.interruptTurn(input).pipe(
1882 + Effect.timeoutOrElse({
1883 + duration: PROVIDER_INTERRUPT_TIMEOUT_MS,
1884 + orElse: () =>
1885 + Effect.fail(
1886 + new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
1887 + driver: runtime.driver,
1888 + providerThreadId: input.providerThread.id,
1889 + providerTurnId: input.providerTurnId,
1890 + timeoutMs: PROVIDER_INTERRUPT_TIMEOUT_MS,
1891 + }),
1892 + ),
1893 + }),
1894 + turnMetrics("interrupt"),
1895 + ),
1896 + ),
1897 ),
1898 respondToRuntimeRequest: (input) =>
1899 observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
@@ -2276,6 +2359,40 @@ export const layerWithOptions = (
2359 ),
2360 ),
2361 release: releaseEntry,
2362 + releaseUnlessBusy: Effect.fn("ProviderSessionManager.releaseUnlessBusy")(function* (input) {
2363 + const key = sessionKey(input.providerSessionId);
2364 + const before = (yield* Ref.get(sessions)).get(key);
2365 + yield* releaseEntry({
2366 + providerSessionId: input.providerSessionId,
2367 + reason: input.reason,
2368 + ...(input.detail === undefined ? {} : { detail: input.detail }),
2369 + keepIfBusyOtherThan: input.ownProviderThreadIds,
2370 + });
2371 + // Identity, not presence: open can install a replacement under the same id meanwhile.
2372 + return before !== undefined && (yield* Ref.get(sessions)).get(key) !== before;
2373 + }),
2374 + otherWorkingThreads: Effect.fn("ProviderSessionManager.otherWorkingThreads")(
2375 + function* (input) {
2376 + const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId));
2377 + if (entry === undefined || !entry.supportsMultipleProviderThreads) return [];
2378 + return yield* Effect.filter(
2379 + [...entry.attachedThreadIds].filter((threadId) => threadId !== input.threadId),
2380 + (threadId) =>
2381 + // Runs, not the shell's activeRunId: the shell omits silent runs such as idle
2382 + // compaction. Background work outlives its run and still needs the process.
2383 + Effect.zipWith(
2384 + projectionStore.getThreadRecords(threadId, ["runs"]),
2385 + projectionStore.getThreadShell(threadId),
2386 + ({ runs }, shell) =>
2387 + runs.some((run) => WORKING_RUN_STATUSES.has(run.status)) ||
2388 + (shell?.pendingBackgroundTasks?.length ?? 0) > 0,
2389 + ).pipe(
2390 + // An unreadable thread may still be working; never kill it unseen.
2391 + Effect.orElseSucceed(() => true),
2392 + ),
2393 + );
2394 + },
2395 + ),
2396 detach: Effect.fn("ProviderSessionManager.detach")(
2397 function* (input) {
2398 const key = sessionKey(input.providerSessionId);
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+2
@@ -284,6 +284,8 @@ it.effect(
284 closeInstance: () => Effect.void,
285 release: () => Effect.void,
286 detach: () => Effect.void,
287 + otherWorkingThreads: () => Effect.succeed([]),
288 + releaseUnlessBusy: () => Effect.succeed(true),
289 }),
290 );
291 const layerControl = ProviderTurnControlService.layer.pipe(
apps/server/src/orchestration-v2/ProviderTurnControlService.ts
+22 -5
@@ -38,6 +38,8 @@ export class ProviderTurnControlError extends Schema.TaggedError<ProviderTurnCon
38 operation: Schema.Literals(["interrupt", "restart", "steer"]),
39 providerTurnId: ProviderTurnId,
40 turnCompleted: Schema.optional(Schema.Boolean),
41 + /** The provider never confirmed the stop; Force stop is the way out. */
42 + unresponsive: Schema.optional(Schema.Boolean),
43 cause: Schema.optional(Schema.Defect()),
44 },
45 ) {}
@@ -218,11 +220,26 @@ export const layer: Layer.Layer<
220 });
221 }
222 // Settled turns still reach adapters, which alone know whether background work remains.
221 - yield* session.value.interruptTurn({
222 - providerThread: loaded.providerThread,
223 - providerTurnId: loaded.providerTurn.id,
224 - requestRuntimeRestart: true,
225 - });
223 + yield* session.value
224 + .interruptTurn({
225 + providerThread: loaded.providerThread,
226 + providerTurnId: loaded.providerTurn.id,
227 + requestRuntimeRestart: true,
228 + })
229 + .pipe(
230 + Effect.catchTags({
231 + ProviderAdapterInterruptTimeoutError: (cause) =>
232 + Effect.fail(
233 + new ProviderTurnControlError({
234 + threadId: input.threadId,
235 + operation: "interrupt",
236 + providerTurnId: input.providerTurnId,
237 + unresponsive: true,
238 + cause,
239 + }),
240 + ),
241 + }),
242 + );
243 // Use real time; ingestion fibers do not advance TestClock and would leave Stop blocked.
244 yield* Effect.gen(function* () {
245 const deadline = (yield* Clock.currentTimeMillis) + 2_000;
apps/server/src/orchestration-v2/ThreadManagementService.test.ts
+173
@@ -27,6 +27,8 @@ import * as Queue from "effect/Queue";
27 import * as Stream from "effect/Stream";
28 import * as TestClock from "effect/testing/TestClock";
29
30 +import { AgentScope } from "@t3tools/shared/AgentScope";
31 +
32 import * as ServerSettings from "../serverSettings.ts";
33 import * as SqlitePersistence from "../persistence/Sqlite.ts";
34 import * as CommandReceiptStore from "./CommandReceiptStore.ts";
@@ -596,3 +598,174 @@ it.effect.each([
598 expect(dispatched).toEqual(settles ? ["thread.settle"] : []);
599 }),
600 );
601 +
602 +it.effect.each([
603 + {
604 + name: "kills the thread's own agent and ends its turn",
605 + others: [] as ReadonlyArray<string>,
606 + killed: true,
607 + expected: { outcome: "killed", sharedWithThreadIds: [] },
608 + teardown: ["release"],
609 + },
610 + {
611 + name: "keeps a process another working thread shares and only ends this turn",
612 + others: ["thread:thread-management:force-stop:other"],
613 + killed: false,
614 + expected: {
615 + outcome: "shared",
616 + sharedWithThreadIds: ["thread:thread-management:force-stop:other"],
617 + },
618 + teardown: ["detach"],
619 + },
620 + {
621 + name: "still settles a finished run's background work in a shared process",
622 + others: ["thread:thread-management:force-stop:other"],
623 + killed: false,
624 + settledRun: true,
625 + expected: {
626 + outcome: "shared",
627 + sharedWithThreadIds: ["thread:thread-management:force-stop:other"],
628 + },
629 + teardown: ["detach"],
630 + },
631 + {
632 + name: "detaches when a sibling starts a turn just before the session is released",
633 + others: [] as ReadonlyArray<string>,
634 + killed: false,
635 + siblingBusy: true,
636 + expected: { outcome: "shared", sharedWithThreadIds: [] },
637 + teardown: ["release", "detach"],
638 + },
639 + {
640 + name: "reports a teardown that has not finished in time as still releasing",
641 + others: [] as ReadonlyArray<string>,
642 + killed: false,
643 + releaseHangs: true,
644 + expected: { outcome: "releasing", sharedWithThreadIds: [] },
645 + teardown: ["release"],
646 + },
647 +])("Force stop $name", ({ others, killed, expected, teardown, ...row }) =>
648 + Effect.gen(function* () {
649 + const threadId = ThreadId.make("thread:thread-management:force-stop");
650 + const providerSessionId = ProviderSessionId.make("provider-session:force-stop");
651 + const providerThreadId = ProviderThreadId.make("provider-thread:force-stop");
652 + const runId = RunId.make("run:thread-management:force-stop");
653 + const dispatched: Array<OrchestrationV2Command | { readonly type: string }> = [];
654 + const teardowns: Array<string> = [];
655 + const tornDown = yield* Deferred.make<void>();
656 + const layerTest = ThreadManagementService.layer.pipe(
657 + Layer.provide(
658 + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
659 + isLive: () => Effect.succeed(true),
660 + otherWorkingThreads: () => Effect.succeed(others.map((id) => ThreadId.make(id))),
661 + releaseUnlessBusy: () =>
662 + Effect.sync(() => teardowns.push("release")).pipe(
663 + Effect.andThen(
664 + "releaseHangs" in row ? Effect.never : Effect.succeed(!("siblingBusy" in row)),
665 + ),
666 + ),
667 + detach: () =>
668 + Effect.sync(() => teardowns.push("detach")).pipe(
669 + Effect.andThen(Deferred.succeed(tornDown, undefined)),
670 + Effect.asVoid,
671 + ),
672 + }),
673 + ),
674 + Layer.provide(
675 + Layer.mock(Orchestrator.OrchestratorV2)({
676 + getThreadRecords: () =>
677 + Effect.succeed({
678 + thread: { id: threadId, deletedAt: null },
679 + runs: [
680 + {
681 + id: runId,
682 + ordinal: 1,
683 + status: "settledRun" in row ? "completed" : "running",
684 + activeAttemptId: "attempt",
685 + },
686 + // A queued follow-up neither supersedes the request nor becomes the target.
687 + { id: RunId.make("run:queued-follow-up"), ordinal: 2, status: "queued" },
688 + ],
689 + providerThreads: [{ id: providerThreadId, appThreadId: threadId, providerSessionId }],
690 + providerTurns: [
691 + {
692 + id: "provider-turn:force-stop",
693 + providerThreadId,
694 + runAttemptId: "attempt",
695 + status: "settledRun" in row ? "completed" : "running",
696 + },
697 + ],
698 + } as unknown as OrchestrationV2ThreadProjection),
699 + dispatch: (command) =>
700 + Effect.sync(() => {
701 + dispatched.push(command);
702 + return { sequence: 1, storedEvents: [] } as never;
703 + }),
704 + }),
705 + ),
706 + Layer.provide(
707 + Layer.succeed(AgentScope, {
708 + ...AgentScope.defaultValue(),
709 + kill: () => Effect.succeed(killed),
710 + }),
711 + ),
712 + Layer.provide(ServerSettings.layerTest()),
713 + Layer.provide(layerReceiptsTest),
714 + );
715 + yield* Effect.gen(function* () {
716 + const service = yield* ThreadManagementService.ThreadManagementService;
717 + const stopping = yield* service.forceStopThread({ threadId, runId }).pipe(Effect.forkChild);
718 + yield* TestClock.adjust("10 seconds");
719 + expect(yield* Fiber.join(stopping)).toEqual(expected);
720 + expect(dispatched).toMatchObject([
721 + {
722 + type: "thread.background-work.settle",
723 + providerThreadId,
724 + reason: "Force stopped by user",
725 + },
726 + ]);
727 + if (teardown.at(-1) === "detach") yield* Deferred.await(tornDown);
728 + expect(teardowns).toEqual(teardown);
729 + }).pipe(Effect.provide(layerTest));
730 + }),
731 +);
732 +
733 +it.effect("Force stop refuses once a newer run has started, before killing anything", () =>
734 + Effect.gen(function* () {
735 + const threadId = ThreadId.make("thread:thread-management:force-stop-superseded");
736 + let killed = false;
737 + const layerTest = ThreadManagementService.layer.pipe(
738 + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
739 + Layer.provide(
740 + Layer.mock(Orchestrator.OrchestratorV2)({
741 + getThreadRecords: () =>
742 + Effect.succeed({
743 + thread: { id: threadId, deletedAt: null },
744 + runs: [
745 + { id: RunId.make("run:stopped"), ordinal: 1, status: "running" },
746 + { id: RunId.make("run:newer"), ordinal: 2, status: "running" },
747 + ],
748 + providerThreads: [],
749 + providerTurns: [],
750 + } as unknown as OrchestrationV2ThreadProjection),
751 + }),
752 + ),
753 + Layer.provide(
754 + Layer.succeed(AgentScope, {
755 + ...AgentScope.defaultValue(),
756 + kill: () => Effect.sync(() => (killed = true)),
757 + }),
758 + ),
759 + Layer.provide(ServerSettings.layerTest()),
760 + Layer.provide(layerReceiptsTest),
761 + );
762 + const error = yield* Effect.gen(function* () {
763 + const service = yield* ThreadManagementService.ThreadManagementService;
764 + return yield* Effect.flip(
765 + service.forceStopThread({ threadId, runId: RunId.make("run:stopped") }),
766 + );
767 + }).pipe(Effect.provide(layerTest));
768 + expect(error.reason).toBe("superseded");
769 + expect(killed).toBe(false);
770 + }),
771 +);
apps/server/src/orchestration-v2/ThreadManagementService.ts
+166
@@ -14,6 +14,9 @@ import {
14 MessageId,
15 OrchestrationV2StopWorkflowError,
16 type OrchestrationV2StopWorkflowInput,
17 + OrchestrationV2ForceStopThreadError,
18 + type OrchestrationV2ForceStopThreadInput,
19 + type OrchestrationV2ForceStopThreadResult,
20 type ModelSelection,
21 type OrchestrationV2Actor,
22 type OrchestrationV2Command,
@@ -30,6 +33,7 @@ import {
33 ProjectId,
34 PROVIDER_SEND_TURN_MAX_FILE_BYTES,
35 ProviderInstanceId,
36 + type ProviderSessionId,
37 resolveProjectProviderInstanceEnabled,
38 RunId,
39 type ScheduledTaskId,
@@ -38,10 +42,13 @@ import {
42 } from "@t3tools/contracts";
43 import { delegatedTaskProgress } from "@t3tools/provider-core/server/subagentProjection";
44 import { threadTranscriptHeader } from "@t3tools/shared/threadTranscript";
45 +import { AgentScope } from "@t3tools/shared/AgentScope";
46 +import * as Clock from "effect/Clock";
47 import * as Context from "effect/Context";
48 import * as DateTime from "effect/DateTime";
49 import * as Duration from "effect/Duration";
50 import * as Effect from "effect/Effect";
51 +import * as Fiber from "effect/Fiber";
52 import * as Layer from "effect/Layer";
53 import * as Option from "effect/Option";
54 import * as Ref from "effect/Ref";
@@ -454,6 +461,13 @@ export interface ThreadManagementServiceShape {
461 readonly stopWorkflow: (
462 input: OrchestrationV2StopWorkflowInput,
463 ) => Effect.Effect<void, OrchestrationV2StopWorkflowError>;
464 + /**
465 + * Kills the thread's agent for a provider that no longer responds, then ends
466 + * its turn as interrupted. Never kills a process other working threads share.
467 + */
468 + readonly forceStopThread: (
469 + input: OrchestrationV2ForceStopThreadInput,
470 + ) => Effect.Effect<OrchestrationV2ForceStopThreadResult, OrchestrationV2ForceStopThreadError>;
471 readonly awaitBackgroundTasks: (input: {
472 readonly threadId: ThreadId;
473 readonly taskIds: ReadonlyArray<string>;
@@ -547,12 +561,16 @@ function latestSteerableRun(
561 }
562
563 const SETTLE_AFTER_RUN_WAIT_MS = 24 * 60 * 60 * 1_000;
564 +const FORCE_STOP_REASON = "Force stopped by user";
565 +const FORCE_STOP_RELEASE_WAIT = "10 seconds";
566 +const isForceStopThreadError = Schema.is(OrchestrationV2ForceStopThreadError);
567
568 const make = Effect.gen(function* () {
569 const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor;
570 const orchestrator = yield* Orchestrator.OrchestratorV2;
571 const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
572 const layerScope = yield* Effect.scope;
573 + const agentScope = yield* AgentScope;
574 const legacyImporter = yield* LegacyV1ThreadImporter.LegacyV1ThreadImporter;
575 const serverSettings = yield* ServerSettings.ServerSettingsService;
576 const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
@@ -1072,6 +1090,153 @@ const make = Effect.gen(function* () {
1090 return { type: "interrupt_requested", run: interruptibleRun, dispatch } as const;
1091 });
1092
1093 + const forceStopThread: ThreadManagementServiceShape["forceStopThread"] = Effect.fn(
1094 + "ThreadManagementService.forceStopThread",
1095 + )(
1096 + function* (input) {
1097 + // Under the thread lock no new run can be created between the check and the kill. The
1098 + // settlement below dispatches, which takes that lock itself, so it runs after release.
1099 + const { records, target, killed } = yield* threadDispatch.withLock(
1100 + input.threadId,
1101 + Effect.gen(function* () {
1102 + const records = yield* orchestrator.getThreadRecords(input.threadId, [
1103 + "runs",
1104 + "providerThreads",
1105 + "providerTurns",
1106 + ]);
1107 + // The run Force stop acts on: the active one, else the latest that started. Queued
1108 + // follow-ups never ran, so they neither count as targets nor supersede the request.
1109 + const target =
1110 + latestActiveRun(records) ??
1111 + records.runs
1112 + .filter((run) => run.status !== "queued")
1113 + .reduce<OrchestrationV2Run | undefined>(
1114 + (current, run) =>
1115 + current === undefined || run.ordinal > current.ordinal ? run : current,
1116 + undefined,
1117 + );
1118 + if (input.runId !== undefined && input.runId !== target?.id) {
1119 + return yield* new OrchestrationV2ForceStopThreadError({
1120 + threadId: input.threadId,
1121 + reason: "superseded",
1122 + });
1123 + }
1124 + // Shared scopes are never tracked per thread, so this only kills the thread's own agent.
1125 + const killed = yield* agentScope.kill(input.threadId);
1126 + return { records, target, killed };
1127 + }),
1128 + );
1129 + const sessionIds = new Set(
1130 + records.providerThreads.flatMap((providerThread) =>
1131 + providerThread.appThreadId === input.threadId && providerThread.providerSessionId !== null
1132 + ? [providerThread.providerSessionId]
1133 + : [],
1134 + ),
1135 + );
1136 + // A run that already ended may still hold background work; settling it is a no-op otherwise.
1137 + const run = target;
1138 + if (run !== undefined) {
1139 + const active = isActiveRun(run);
1140 + const providerTurn = records.providerTurns.findLast(
1141 + (turn) =>
1142 + turn.runAttemptId === run.activeAttemptId && (!active || turn.status === "running"),
1143 + );
1144 + const commandId = CommandId.make(
1145 + `server:force-stop:${run.id}:${yield* Clock.currentTimeMillis}`,
1146 + );
1147 + yield* providerTurn === undefined && !active
1148 + ? Effect.void
1149 + : providerTurn === undefined
1150 + ? orchestrator.dispatch({
1151 + type: "run.interrupt",
1152 + commandId,
1153 + threadId: input.threadId,
1154 + runId: run.id,
1155 + reason: FORCE_STOP_REASON,
1156 + })
1157 + : orchestrator.dispatch({
1158 + type: "thread.background-work.settle",
1159 + commandId,
1160 + threadId: input.threadId,
1161 + providerThreadId: providerTurn.providerThreadId,
1162 + providerTurnId: providerTurn.id,
1163 + reason: FORCE_STOP_REASON,
1164 + });
1165 + }
1166 + const ownProviderThreadIds = records.providerThreads.flatMap((providerThread) =>
1167 + providerThread.appThreadId === input.threadId ? [providerThread.id] : [],
1168 + );
1169 + const detach = (providerSessionId: ProviderSessionId) =>
1170 + sessions
1171 + .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON })
1172 + .pipe(
1173 + Effect.catchCause((cause) =>
1174 + Effect.logWarning("orchestration-v2.force-stop-detach-failed", {
1175 + threadId: input.threadId,
1176 + cause,
1177 + }),
1178 + ),
1179 + Effect.forkIn(layerScope),
1180 + );
1181 + let shared = false;
1182 + const sharedWith = new Set<ThreadId>();
1183 + const releases: Array<{
1184 + readonly providerSessionId: ProviderSessionId;
1185 + readonly fiber: Fiber.Fiber<boolean, ProviderSessionManager.ProviderSessionManagerV2Error>;
1186 + }> = [];
1187 + for (const providerSessionId of sessionIds) {
1188 + if (!(yield* sessions.isLive(providerSessionId))) continue;
1189 + // Decided just before teardown, so the reply matches what happens to the process.
1190 + const others = yield* sessions.otherWorkingThreads({
1191 + providerSessionId,
1192 + threadId: input.threadId,
1193 + });
1194 + for (const threadId of others) sharedWith.add(threadId);
1195 + if (others.length > 0) {
1196 + shared = true;
1197 + yield* detach(providerSessionId);
1198 + continue;
1199 + }
1200 + // The session manager re-checks atomically with turn starts and keeps a session a
1201 + // sibling began using in the meantime.
1202 + const fiber = yield* sessions
1203 + .releaseUnlessBusy({
1204 + providerSessionId,
1205 + ownProviderThreadIds,
1206 + reason: "manual_shutdown",
1207 + detail: FORCE_STOP_REASON,
1208 + })
1209 + .pipe(Effect.forkIn(layerScope));
1210 + releases.push({ providerSessionId, fiber });
1211 + }
1212 + // Teardown waits on the provider's own exit paths; past this bound it keeps going on its own.
1213 + let releasing = false;
1214 + for (const release of releases) {
1215 + const released = yield* Fiber.join(release.fiber).pipe(
1216 + Effect.timeoutOption(FORCE_STOP_RELEASE_WAIT),
1217 + );
1218 + if (Option.isNone(released)) {
1219 + releasing = true;
1220 + } else if (!released.value) {
1221 + shared = true;
1222 + yield* detach(release.providerSessionId);
1223 + }
1224 + }
1225 + return {
1226 + outcome: killed ? "killed" : shared ? "shared" : releasing ? "releasing" : "released",
1227 + sharedWithThreadIds: [...sharedWith],
1228 + } as const;
1229 + },
1230 + (effect, input) =>
1231 + effect.pipe(
1232 + Effect.mapError((cause) =>
1233 + isForceStopThreadError(cause)
1234 + ? cause
1235 + : new OrchestrationV2ForceStopThreadError({ threadId: input.threadId, cause }),
1236 + ),
1237 + ),
1238 + );
1239 +
1240 const stopWorkflow: ThreadManagementServiceShape["stopWorkflow"] = Effect.fn(
1241 "ThreadManagementService.stopWorkflow",
1242 )(
@@ -1354,6 +1519,7 @@ const make = Effect.gen(function* () {
1519 interruptThread,
1520 stopDelegatedTasks,
1521 stopWorkflow,
1522 + forceStopThread,
1523 awaitBackgroundTasks,
1524 getThreadEventSequence: orchestrator.getThreadEventSequence,
1525 recoverDelegatedTask: orchestrator.recoverDelegatedTask,
apps/server/src/orchestration-v2/ThreadStop.test.ts
+112
@@ -1552,3 +1552,115 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)
1552 );
1553 }).pipe(Effect.provide(layerTest)),
1554 );
1555 +
1556 +it.effect("an unanswered Stop is flagged until Force stop ends the run with its own reason", () =>
1557 + Effect.gen(function* () {
1558 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1559 + const projections = yield* ProjectionStore.ProjectionStoreV2;
1560 + const threadId = ThreadId.make("thread:force-stop");
1561 + yield* createWatchingThread(threadId, 6);
1562 + yield* send(threadId, "work", "start_immediately");
1563 + const now = yield* DateTime.now;
1564 + const started = yield* orchestrator.getThreadProjection(threadId);
1565 + const run = started.runs[0]!;
1566 + const providerThreadId = run.providerThreadId!;
1567 + const providerTurnId = ProviderTurnId.make("provider-turn:force-stop");
1568 + const apply = (event: Omit<OrchestrationV2DomainEvent, "id" | "threadId">) =>
1569 + projections.apply({
1570 + ...event,
1571 + id: EventId.make(`event:force-stop:${event.type}`),
1572 + threadId,
1573 + } as OrchestrationV2DomainEvent);
1574 + const attempt = started.attempts[0]!;
1575 + const attemptId = attempt.id;
1576 + yield* apply({
1577 + type: "run.updated",
1578 + runId: run.id,
1579 + occurredAt: now,
1580 + payload: { ...run, status: "running", startedAt: now, activeAttemptId: attemptId },
1581 + });
1582 + yield* apply({
1583 + type: "run-attempt.updated",
1584 + occurredAt: now,
1585 + payload: { ...attempt, status: "running", startedAt: now },
1586 + });
1587 + yield* apply({
1588 + type: "provider-turn.updated",
1589 + occurredAt: now,
1590 + payload: {
1591 + id: providerTurnId,
1592 + providerThreadId,
1593 + nodeId: run.rootNodeId!,
1594 + runAttemptId: attemptId,
1595 + nativeTurnRef: null,
1596 + ordinal: 1,
1597 + status: "running",
1598 + startedAt: now,
1599 + completedAt: null,
1600 + },
1601 + });
1602 + yield* apply({
1603 + type: "turn-item.updated",
1604 + runId: run.id,
1605 + occurredAt: now,
1606 + payload: {
1607 + id: TurnItemId.make("turn-item:force-stop:interrupt-request"),
1608 + threadId,
1609 + runId: run.id,
1610 + nodeId: run.rootNodeId!,
1611 + providerThreadId,
1612 + providerTurnId,
1613 + nativeItemRef: null,
1614 + parentItemId: null,
1615 + ordinal: 1_000,
1616 + status: "completed",
1617 + title: "Interrupt requested",
1618 + startedAt: now,
1619 + completedAt: now,
1620 + updatedAt: now,
1621 + type: "run_interrupt_request",
1622 + message: "Stopped by user",
1623 + },
1624 + });
1625 + const target = { threadId, providerThreadId, providerTurnId };
1626 + const request = Effect.map(orchestrator.getThreadProjection(threadId), (projection) =>
1627 + projection.turnItems.find((item) => item.type === "run_interrupt_request"),
1628 + );
1629 +
1630 + yield* orchestrator.dispatch({
1631 + ...target,
1632 + type: "thread.interrupt.unresponsive",
1633 + commandId: CommandId.make("force-stop:unresponsive"),
1634 + });
1635 + const flagged = yield* request;
1636 + assert.isTrue(
1637 + flagged?.type === "run_interrupt_request" && flagged.unresponsiveAt !== undefined,
1638 + );
1639 + assert.deepEqual((yield* threadState(threadId)).runs, ["running"]);
1640 +
1641 + yield* orchestrator.dispatch({
1642 + ...target,
1643 + type: "thread.background-work.settle",
1644 + commandId: CommandId.make("force-stop:settle"),
1645 + reason: "Force stopped by user",
1646 + });
1647 + const settled = yield* orchestrator.getThreadProjection(threadId);
1648 + assert.deepEqual(
1649 + settled.runs.map((candidate) => candidate.status),
1650 + ["interrupted"],
1651 + );
1652 + const result = settled.turnItems.find((item) => item.type === "run_interrupt_result");
1653 + assert.equal(
1654 + result?.type === "run_interrupt_result" ? result.message : undefined,
1655 + "Force stopped by user",
1656 + );
1657 +
1658 + // A provider confirming late changes nothing once the run has ended.
1659 + const late = yield* orchestrator.dispatch({
1660 + ...target,
1661 + type: "thread.interrupt.unresponsive",
1662 + commandId: CommandId.make("force-stop:late-unresponsive"),
1663 + });
1664 + assert.lengthOf(late.storedEvents, 0);
1665 + }).pipe(Effect.provide(layerTest)),
1666 +);
apps/server/src/process/agentScope.test.ts
+32
@@ -110,6 +110,8 @@ const withFakeSystemd = <A, E>(
110 ...fileSystem,
111 readFileString: (path, encoding) =>
112 fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
113 + writeFileString: (path, data, options) =>
114 + fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
115 }),
116 );
117 const { stateDir } = yield* ServerConfig.ServerConfig;
@@ -213,6 +215,36 @@ describe("AgentScope service", () => {
215 ),
216 );
217
218 + it.effect.each([{ oomPolicy: true }, { oomPolicy: false }])(
219 + "Force stop kills a thread's own scope but never a shared Codex scope (OOM policy $oomPolicy)",
220 + ({ oomPolicy }) =>
221 + withFakeSystemd(
222 + ({ scope, units, calls, cgroupRoot }) =>
223 + Effect.gen(function* () {
224 + const agent = yield* scope.wrap({
225 + command: "agent",
226 + args: [],
227 + name: "t",
228 + threadId: "a",
229 + });
230 + exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
231 + const codex = yield* launchCodex(scope, "b");
232 + exitedAgent(units, cgroupRoot, unitOf(codex), { populated: true, oomKills: 0 });
233 +
234 + expect(yield* scope.kill("a")).toBe(true);
235 + expect(
236 + NodeFS.readFileSync(NodePath.join(cgroupRoot, unitOf(agent), "cgroup.kill"), "utf8"),
237 + ).toBe("1");
238 + expect(yield* scope.kill("b")).toBe(false);
239 + expect(NodeFS.existsSync(NodePath.join(cgroupRoot, unitOf(codex), "cgroup.kill"))).toBe(
240 + false,
241 + );
242 + expect(calls.filter(([, verb]) => verb === "kill")).toEqual([]);
243 + }),
244 + { oomPolicy },
245 + ),
246 + );
247 +
248 it.effect("keeps a running scope while hundreds of other threads launch", () =>
249 withFakeSystemd(({ scope, units }) =>
250 Effect.gen(function* () {
apps/server/src/process/agentScope.ts
+41 -3
@@ -215,7 +215,8 @@ export const make = Effect.gen(function* () {
215 }),
216 );
217
218 - const scopeByThread = new Map<string, { unit: string; oomKilled: boolean }>();
218 + // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd.
219 + const scopeByThread = new Map<string, { unit: string; oomKilled: boolean; shared: boolean }>();
220
221 const readScope = Effect.fn("agentScope.readScope")(function* (systemctl: string, unit: string) {
222 const shown = yield* run(systemctl, [
@@ -277,10 +278,11 @@ export const make = Effect.gen(function* () {
278 systemctl: string,
279 threadId: string,
280 unit: string | undefined,
281 + shared = false,
282 ) {
283 const previous = scopeByThread.get(threadId);
284 scopeByThread.delete(threadId);
283 - if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false });
285 + if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared });
286 if (previous !== undefined) yield* forget(systemctl, previous.unit);
287 if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl);
288 });
@@ -340,6 +342,40 @@ export const make = Effect.gen(function* () {
342 }
343 });
344
345 + const kill: AgentScopeShape["kill"] = Effect.fn("AgentScope.kill")(function* (threadId) {
346 + const tracked = scopeByThread.get(threadId);
347 + if (tracked === undefined || tracked.shared) return false;
348 + const scope = yield* systemd;
349 + if (scope === undefined) return false;
350 + const shown = yield* run(scope.systemctl, [
351 + "--user",
352 + "show",
353 + tracked.unit,
354 + "--property=ActiveState,ControlGroup",
355 + ]);
356 + if (shown._tag === "None" || shown.value.code !== 0) return false;
357 + const values = parseKeyValues(shown.value.stdout);
358 + if (values.get("ActiveState") !== "active") return false;
359 + const controlGroup = values.get("ControlGroup") ?? "";
360 + // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback.
361 + const killedByCgroup =
362 + controlGroup.length > 0 &&
363 + (yield* fileSystem
364 + .writeFileString(path.join("/sys/fs/cgroup", controlGroup, "cgroup.kill"), "1")
365 + .pipe(
366 + Effect.as(true),
367 + Effect.orElseSucceed(() => false),
368 + ));
369 + if (killedByCgroup) return true;
370 + const killed = yield* run(scope.systemctl, [
371 + "--user",
372 + "kill",
373 + "--signal=SIGKILL",
374 + tracked.unit,
375 + ]);
376 + return killed._tag === "Some" && killed.value.code === 0;
377 + });
378 +
379 const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) {
380 if (!scopeByThread.has(threadId)) return;
381 const scope = yield* systemd;
@@ -365,7 +401,8 @@ export const make = Effect.gen(function* () {
401 // Clear the launch thread's previous scope instead.
402 const continuing = shared === true && scope.supportsOomPolicy;
403 const owned = threadId !== undefined && !continuing;
368 - if (threadId !== undefined) yield* track(scope.systemctl, threadId, owned ? unit : undefined);
404 + if (threadId !== undefined)
405 + yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true);
406 const systemdScope: AgentSystemdScope = {
407 systemdRun: scope.systemdRun,
408 runtimeDir: scope.runtimeDir,
@@ -385,6 +422,7 @@ export const make = Effect.gen(function* () {
422 }),
423 oomKilled,
424 stop,
425 + kill,
426 clear,
427 } satisfies AgentScopeShape;
428 });
apps/server/src/relay/AgentAwarenessRelay.test.ts
+1
@@ -219,6 +219,7 @@ const makeTestRelay = Effect.fnUntraced(function* (
219 settleThread: unused,
220 interruptThread: unused,
221 stopWorkflow: () => Effect.die("unused stopWorkflow"),
222 + forceStopThread: () => Effect.die("unused forceStopThread"),
223 awaitBackgroundTasks: unused,
224 stopDelegatedTasks: unused,
225 getThreadEventSequence: unused,
apps/server/src/ws.ts
+2
@@ -1914,6 +1914,8 @@ export const layerWsRpc = (
1914 ),
1915 ),
1916 [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: (input) => threadManagement.stopWorkflow(input),
1917 + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: (input) =>
1918 + threadManagement.forceStopThread(input),
1919 [ORCHESTRATION_V2_WS_METHODS.getWorkflowScript]: (input) =>
1920 readWorkflowScript({ scriptPath: input.scriptPath }),
1921 [ORCHESTRATION_V2_WS_METHODS.getTurnItem]: (input) =>
apps/web/src/components/ChatView.tsx
+5 -7
@@ -2349,7 +2349,8 @@ export default function ChatView(props: ChatViewProps) {
2349 const {
2350 acknowledgeActiveThreadWoke,
2351 activeBranchMismatchKey,
2352 - backgroundWorkNow,
2352 + attentionNow,
2353 + workNow,
2354 childInputNow,
2355 goalNow,
2356 handleSwitchCheckoutToThread,
@@ -2993,10 +2994,7 @@ export default function ChatView(props: ChatViewProps) {
2994 const onExpandTimelineImageStable = useStableCallback(onExpandTimelineImage);
2995 const shownThreadSyncPhase = activeEnvironmentUnavailable ? null : delayedThreadSyncPhase;
2996 const hasThreadNow =
2996 - childInputNow !== null ||
2997 - backgroundWorkNow !== null ||
2998 - goalNow !== null ||
2999 - shownThreadSyncPhase !== null;
2997 + attentionNow !== null || workNow !== null || goalNow !== null || shownThreadSyncPhase !== null;
2998 const queuedRunCount = serverProjection ? getUserQueuedThreadRuns(serverProjection).length : 0;
2999 // The queue control is mounted for every server thread but joins the footer only while it shows
3000 // something, by its own rule: an optimistic message stops counting once the server has it.
@@ -3035,8 +3033,8 @@ export default function ChatView(props: ChatViewProps) {
3033 )}
3034 <ThreadNow
3035 key={routeThreadKey}
3038 - attention={childInputNow}
3039 - work={backgroundWorkNow}
3036 + attention={attentionNow}
3037 + work={workNow}
3038 syncPhase={shownThreadSyncPhase}
3039 goal={goalNow}
3040 />
apps/web/src/components/chat/ThreadActivityBanners.tsx
+43
@@ -7,17 +7,20 @@ import type { EnvironmentThreadShell, ThreadStructure } from "@t3tools/client-ru
7 import {
8 presentPendingBackgroundWork,
9 presentProviderGoal,
10 + type RunStopPresentation,
11 } from "@t3tools/client-runtime/state/thread-execution";
12 import { childThreadInputsTitle } from "@t3tools/client-runtime/state/threads";
13 import { formatDuration } from "@t3tools/shared/orchestrationTiming";
14 import * as DateTime from "effect/DateTime";
15 import {
16 CircleDashedIcon,
17 + CircleStopIcon,
18 GitBranchIcon,
19 MessageCircleIcon,
20 ShieldQuestionIcon,
21 TargetIcon,
22 TerminalIcon,
23 + TriangleAlertIcon,
24 } from "lucide-react";
25 import { Fragment } from "react";
26 import { tickTextContent } from "~/lib/tickTextContent";
@@ -105,6 +108,46 @@ export function goalNowItem(input: {
108 };
109 }
110
111 +/** A Stop the provider has not finished; Force stop once it stops answering. */
112 +export function runStopNowItem(input: {
113 + readonly threadId: ThreadId;
114 + readonly stop: RunStopPresentation;
115 + readonly canForceStop: boolean;
116 + readonly forceStopping: boolean;
117 + readonly onForceStop: () => void;
118 +}): ThreadNowItem {
119 + const unresponsive = input.stop.state === "unresponsive";
120 + const requestedAtMs = Date.parse(input.stop.requestedAt);
121 + return {
122 + id: `run-stop:${input.threadId}`,
123 + icon: unresponsive ? (
124 + <TriangleAlertIcon className="text-warning-foreground" />
125 + ) : (
126 + <CircleStopIcon />
127 + ),
128 + title: unresponsive ? "Agent not responding to Stop" : "Stopping",
129 + description: unresponsive
130 + ? "It may be short on memory. Force stop kills its processes."
131 + : undefined,
132 + actions: (
133 + <>
134 + <span
135 + ref={tickTextContent(
136 + (nowMs) => formatDuration(Math.max(1_000, nowMs - requestedAtMs)),
137 + true,
138 + )}
139 + className="tabular-nums"
140 + />
141 + {unresponsive && input.canForceStop ? (
142 + <InlineButton disabled={input.forceStopping} onClick={input.onForceStop}>
143 + {input.forceStopping ? "Force stopping..." : "Force stop"}
144 + </InlineButton>
145 + ) : null}
146 + </>
147 + ),
148 + };
149 +}
150 +
151 /** Background commands and subagents, for the end of the timeline. */
152 export function backgroundWorkNowItem(input: {
153 readonly threadId: ThreadId;
apps/web/src/components/chat/useForceStopThread.ts new
+59
@@ -0,0 +1,59 @@
1 +import { useAtomValue } from "@effect/atom-react";
2 +import type { EnvironmentId, RunId, ThreadId } from "@t3tools/contracts";
3 +import { useState } from "react";
4 +
5 +import { useServerConfigs } from "../../state/entities";
6 +import { orchestrationEnvironment } from "../../state/orchestration";
7 +import {
8 + isAtomCommandInterrupted,
9 + squashAtomCommandFailure,
10 +} from "@t3tools/client-runtime/state/runtime";
11 +import { useAtomCommand } from "../../state/use-atom-command";
12 +import { toastManager } from "../ui/toast";
13 +
14 +/** Force stop for an agent that no longer answers Stop; shared by every place that offers it. */
15 +export function useForceStopThread(environmentId: EnvironmentId) {
16 + const forceStop = useAtomCommand(orchestrationEnvironment.forceStopThread);
17 + const permitted = useAtomValue(
18 + orchestrationEnvironment.forceStopThread.permissionAtom(environmentId),
19 + );
20 + const supported =
21 + useServerConfigs().get(environmentId)?.environment.capabilities.threadForceStop === true;
22 + const canForceStop = permitted && supported;
23 + const [stoppingThreadId, setStoppingThreadId] = useState<ThreadId | null>(null);
24 + /** Pass the run that ignored Stop, so a Force stop left open cannot hit a newer run. */
25 + const forceStopThread = async (threadId: ThreadId, runId?: RunId) => {
26 + if (!canForceStop || stoppingThreadId !== null) return;
27 + setStoppingThreadId(threadId);
28 + const result = await forceStop({
29 + environmentId,
30 + input: runId === undefined ? { threadId } : { threadId, runId },
31 + });
32 + setStoppingThreadId(null);
33 + if (result._tag === "Failure") {
34 + if (isAtomCommandInterrupted(result)) return;
35 + const error = squashAtomCommandFailure(result);
36 + toastManager.add({
37 + type: "error",
38 + title: "Could not force stop the agent",
39 + description: error instanceof Error ? error.message : undefined,
40 + });
41 + return;
42 + }
43 + const others = result.value.sharedWithThreadIds.length;
44 + if (result.value.outcome === "releasing") {
45 + toastManager.add({
46 + type: "info",
47 + title: "Turn stopped; the agent is still shutting down",
48 + });
49 + }
50 + if (result.value.outcome === "shared") {
51 + toastManager.add({
52 + type: "info",
53 + title: "Turn stopped; the agent process kept running",
54 + description: `It is shared with ${others === 0 ? "other" : others} working ${others === 1 ? "thread" : "threads"}. Stop those to free it.`,
55 + });
56 + }
57 + };
58 + return { canForceStop, stoppingThreadId, forceStopThread };
59 +}
apps/web/src/components/chat/useThreadLifecycle.tsx
+33 -2
@@ -58,9 +58,22 @@ import { appAtomRegistry } from "../../rpc/atomRegistry";
58 import { readEnvironmentScope } from "../../state/session";
59 import { useAtomCommand } from "../../state/use-atom-command";
60 import { useOrchestrationCommand } from "../../state/use-orchestration-command";
61 -import { backgroundWorkNowItem, childInputNowItem, goalNowItem } from "./ThreadActivityBanners";
61 +import {
62 + backgroundWorkNowItem,
63 + childInputNowItem,
64 + goalNowItem,
65 + runStopNowItem,
66 +} from "./ThreadActivityBanners";
67 +import { useForceStopThread } from "./useForceStopThread";
68 +import { useAtomValue } from "@effect/atom-react";
69 +import type { RunStopPresentation } from "@t3tools/client-runtime/state/thread-execution";
70 +import { Atom } from "effect/reactivity";
71 +import { environmentThreadDetails } from "../../state/threads";
72 import type { useLocalDispatchState } from "./useLocalDispatchState";
73 const RELATED_THREAD_SHELL_WAIT_MS = 1_000;
74 +const NO_RUN_STOP_ATOM = Atom.make<RunStopPresentation | null>(null).pipe(
75 + Atom.withLabel("web-thread-run-stop:empty"),
76 +);
77
78 type PendingThreadRequests = ReturnType<typeof usePendingThreadRequests>;
79 type LocalDispatchState = ReturnType<typeof useLocalDispatchState>;
@@ -509,6 +522,22 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) {
522 })
523 : null;
524
525 + const forceStop = useForceStopThread(environmentId);
526 + const anyRunStop = useAtomValue(
527 + activeThreadRef ? environmentThreadDetails.runStopAtom(activeThreadRef) : NO_RUN_STOP_ATOM,
528 + );
529 + // A background Stop matters only while that work is still pending.
530 + const runStop = anyRunStop?.background && activeBackgroundTasks.length === 0 ? null : anyRunStop;
531 + const runStopNow =
532 + activeThread && runStop !== null
533 + ? runStopNowItem({
534 + threadId: activeThread.id,
535 + stop: runStop,
536 + canForceStop: forceStop.canForceStop,
537 + forceStopping: forceStop.stoppingThreadId === activeThread.id,
538 + onForceStop: () => void forceStop.forceStopThread(activeThread.id, runStop.runId),
539 + })
540 + : null;
541 const backgroundWorkNow = activeThread
542 ? backgroundWorkNowItem({
543 threadId: activeThread.id,
@@ -592,7 +621,9 @@ export function useThreadLifecycle(input: ThreadLifecycleInput) {
621 shortcuts,
622 acknowledgeActiveThreadWoke,
623 activeBranchMismatchKey,
595 - backgroundWorkNow,
624 + // A Stop that went unanswered needs the user; one in progress is the thread's work.
625 + attentionNow: runStop?.state === "unresponsive" ? runStopNow : childInputNow,
626 + workNow: runStop?.state === "stopping" ? runStopNow : backgroundWorkNow,
627 childInputNow,
628 goalNow,
629 handleSwitchCheckoutToThread,
packages/client-runtime/src/state/orchestration.ts
+4
@@ -107,6 +107,10 @@ export function createOrchestrationEnvironmentAtoms<R, E>(
107 label: "environment-data:orchestration:stop-workflow",
108 tag: ORCHESTRATION_V2_WS_METHODS.stopWorkflow,
109 }),
110 + forceStopThread: createEnvironmentRpcCommand(runtime, {
111 + label: "environment-data:orchestration:force-stop-thread",
112 + tag: ORCHESTRATION_V2_WS_METHODS.forceStopThread,
113 + }),
114 turnItem: createEnvironmentRpcQueryAtomFamily(runtime, {
115 label: "environment-data:orchestration:turn-item",
116 tag: ORCHESTRATION_V2_WS_METHODS.getTurnItem,
packages/client-runtime/src/state/threadDetail.ts
+28
@@ -14,6 +14,7 @@ import {
14 type ThreadQueueWorkflowState,
15 } from "./threadWorkflows.ts";
16 import { deriveThreadTurnSubagents, type ThreadTurnSubagents } from "./threadSubagents.ts";
17 +import { presentRunStop, type RunStopPresentation } from "./threadExecution.ts";
18 import {
19 deriveThreadCheckpointSummaries,
20 type ThreadCheckpointSummary,
@@ -294,6 +295,32 @@ export function createEnvironmentThreadDetailAtoms<E>(
295 return value;
296 }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-queue:${key}`));
297 });
298 + const runStopAtomFamily = Atom.family((key: string) => {
299 + let previous: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems"> | null = null;
300 + let stop: RunStopPresentation | null = null;
301 + return Atom.make((get) => {
302 + const projection = Option.getOrNull(get(threadStateValueAtomFamily(key)).data);
303 + if (projection === null) {
304 + previous = null;
305 + stop = null;
306 + } else if (
307 + projection.runs !== previous?.runs ||
308 + projection.turnItems !== previous?.turnItems
309 + ) {
310 + const { runs, turnItems } = projection;
311 + previous = { runs, turnItems };
312 + const next = presentRunStop(previous);
313 + if (
314 + next?.state !== stop?.state ||
315 + next?.runId !== stop?.runId ||
316 + next?.requestedAt !== stop?.requestedAt ||
317 + next?.background !== stop?.background
318 + )
319 + stop = next;
320 + }
321 + return stop;
322 + }).pipe(Atom.setIdleTTL(0), Atom.withLabel(`environment-thread-run-stop:${key}`));
323 + });
324 const queuedCountAtomFamily = Atom.family((key: string) => {
325 let previous: Pick<OrchestrationV2ThreadProjection, "runs" | "messages"> | null = null;
326 let count = 0;
@@ -435,6 +462,7 @@ export function createEnvironmentThreadDetailAtoms<E>(
462 pendingRequestsAtom: (ref: ScopedThreadRef) => pendingRequestsAtomFamily(threadKey(ref)),
463 queueWorkflowAtom: (ref: ScopedThreadRef) => queueWorkflowAtomFamily(threadKey(ref)),
464 queuedCountAtom: (ref: ScopedThreadRef) => queuedCountAtomFamily(threadKey(ref)),
465 + runStopAtom: (ref: ScopedThreadRef) => runStopAtomFamily(threadKey(ref)),
466 turnSubagentsAtom: (ref: ScopedThreadRef) => turnSubagentsAtomFamily(threadKey(ref)),
467 checkpointSummariesAtom: (ref: ScopedThreadRef) =>
468 checkpointSummariesAtomFamily(threadKey(ref)),
packages/client-runtime/src/state/threadExecution.test.ts
+69
@@ -25,6 +25,7 @@ import {
25 deriveRunlessWorkStartedAt,
26 deriveThreadActivityRun,
27 deriveThreadRuntime,
28 + presentRunStop,
29 threadRuntimeHasInterruptibleRun,
30 } from "./threadExecution.ts";
31
@@ -259,6 +260,74 @@ describe("thread execution presentation", () => {
260 });
261 });
262
263 +describe("presentRunStop", () => {
264 + const signal = (type: "run_interrupt_request" | "run_interrupt_result", runId: RunId) => ({
265 + id: TurnItemId.make(`${type}:${runId}`),
266 + threadId: v2Projection.thread.id,
267 + runId,
268 + nodeId: null,
269 + providerThreadId: null,
270 + providerTurnId: null,
271 + nativeItemRef: null,
272 + parentItemId: null,
273 + ordinal: 1,
274 + status: type === "run_interrupt_result" ? ("interrupted" as const) : ("completed" as const),
275 + title: "Interrupt",
276 + startedAt: now,
277 + completedAt: now,
278 + updatedAt: now,
279 + type,
280 + message: "Stopped by user",
281 + });
282 + const running = run("run-stopping", 1, "running");
283 + const request = signal("run_interrupt_request", running.id);
284 +
285 + it.each([
286 + { name: "no Stop", runs: [running], turnItems: [], state: null },
287 + {
288 + name: "a Stop the agent is winding down",
289 + runs: [running],
290 + turnItems: [request],
291 + state: "stopping",
292 + },
293 + {
294 + name: "a Stop the provider never confirmed",
295 + runs: [running],
296 + turnItems: [{ ...request, unresponsiveAt: now }],
297 + state: "unresponsive",
298 + },
299 + {
300 + name: "a Stop that ended the run",
301 + runs: [running],
302 + turnItems: [request, signal("run_interrupt_result", running.id)],
303 + state: null,
304 + },
305 + {
306 + name: "a settled run's background work still stopping",
307 + runs: [{ ...running, status: "completed" as const, completedAt: now }],
308 + turnItems: [request],
309 + state: null,
310 + },
311 + {
312 + name: "a settled run whose background work ignored Stop",
313 + runs: [{ ...running, status: "completed" as const, completedAt: now }],
314 + turnItems: [{ ...request, unresponsiveAt: now }],
315 + state: "unresponsive",
316 + },
317 + {
318 + name: "a settled run whose background work ignored Stop, with a queued follow-up",
319 + runs: [
320 + { ...running, status: "completed" as const, completedAt: now },
321 + run("run-queued", 2, "queued"),
322 + ],
323 + turnItems: [{ ...request, unresponsiveAt: now }],
324 + state: "unresponsive",
325 + },
326 + ])("presents $name", ({ runs, turnItems, state }) => {
327 + expect(presentRunStop({ runs, turnItems })?.state ?? null).toBe(state);
328 + });
329 +});
330 +
331 describe("deriveRunlessWorkStartedAt", () => {
332 const later = DateTime.makeUnsafe("2026-07-28T10:05:00.000Z");
333 const rootTurn = (
packages/client-runtime/src/state/threadExecution.ts
+49
@@ -294,6 +294,55 @@ const BACKGROUND_WORK_KINDS: Record<
294 background_task: { order: 3, singular: "background task", plural: "background tasks" },
295 };
296
297 +export interface RunStopPresentation {
298 + /** `unresponsive`: the provider did not confirm the stop in time, so Force stop is offered. */
299 + readonly state: "stopping" | "unresponsive";
300 + readonly runId: ThreadStructure["runs"][number]["id"];
301 + readonly requestedAt: string;
302 + /** The run already ended; the Stop targeted its background work. */
303 + readonly background: boolean;
304 +}
305 +
306 +/**
307 + * A Stop the provider has not finished yet. It stays `stopping` while the agent winds down,
308 + * since some providers confirm the request before the turn actually ends. On a run that
309 + * already ended, only an unanswered Stop of its background work counts.
310 + */
311 +export function presentRunStop(
312 + projection: Pick<ThreadStructure, "runs" | "turnItems">,
313 +): RunStopPresentation | null {
314 + let active: ThreadStructure["runs"][number] | null = null;
315 + let latest: ThreadStructure["runs"][number] | null = null;
316 + for (const candidate of projection.runs) {
317 + // Queued follow-ups never ran, so they hold no Stop of their own.
318 + if (candidate.status !== "queued" && candidate.ordinal > (latest?.ordinal ?? -1))
319 + latest = candidate;
320 + if (
321 + INTERRUPTIBLE_RUN_STATUSES.has(candidate.status) &&
322 + candidate.ordinal > (active?.ordinal ?? -1)
323 + )
324 + active = candidate;
325 + }
326 + const run = active ?? latest;
327 + if (run === null) return null;
328 + let request:
329 + | Extract<ThreadStructure["turnItems"][number], { type: "run_interrupt_request" }>
330 + | undefined;
331 + for (const item of projection.turnItems) {
332 + if (item.runId !== run.id) continue;
333 + if (item.type === "run_interrupt_result") return null;
334 + if (item.type === "run_interrupt_request") request = item;
335 + }
336 + if (request === undefined || (active === null && request.unresponsiveAt === undefined))
337 + return null;
338 + return {
339 + state: request.unresponsiveAt === undefined ? "stopping" : "unresponsive",
340 + runId: run.id,
341 + requestedAt: DateTime.formatIso(request.startedAt ?? request.updatedAt),
342 + background: active === null,
343 + };
344 +}
345 +
346 export interface PendingBackgroundWorkItem {
347 readonly taskId: string;
348 readonly kind: BackgroundWorkKind;
packages/contracts/src/clientRpcPermissions.ts
+2
@@ -60,6 +60,8 @@ export const CLIENT_GUARDED_RPC_SCOPES = {
60 [WS_METHODS.scheduledTasksRunNow]: AuthOrchestrationOperateScope,
61 [WS_METHODS.scheduledTasksRotateWebhookToken]: AuthOrchestrationOperateScope,
62 [ORCHESTRATION_V2_WS_METHODS.stopWorkflow]: AuthOrchestrationOperateScope,
63 + // Force stop kills agent processes, as Stop on a terminal does.
64 + [ORCHESTRATION_V2_WS_METHODS.forceStopThread]: AuthOrchestrationOperateScope,
65 // A guest link hands thread access to someone outside, like a pairing link.
66 [WS_METHODS.guestShareCreate]: AuthAccessWriteScope,
67 [WS_METHODS.peerLinksLink]: AuthAccessWriteScope,
packages/contracts/src/environment.ts
+2
@@ -99,6 +99,8 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({
99 /** Hide the switch when absent; older servers silently drop per-project provider patches. */
100 projectProviderInstanceOverrides: Schema.optionalKey(Schema.Boolean),
101 threadSnooze: Schema.optionalKey(Schema.Boolean),
102 + /** Serves `orchestration.forceStopThread`; hide Force stop where it is absent. */
103 + threadForceStop: Schema.optionalKey(Schema.Boolean),
104 /** Every protocol-2 server sends this; older mobile builds still read it. */
105 environmentThemes: Schema.optionalKey(Schema.Boolean),
106 usageLimitSources: Schema.optionalKey(Schema.Boolean),
packages/contracts/src/orchestrationV2.ts
+53
@@ -1676,6 +1676,8 @@ export const OrchestrationV2TurnItem = Schema.Union([
1676 ...OrchestrationV2TurnItemBaseFields,
1677 type: Schema.Literal("run_interrupt_request"),
1678 message: Schema.String,
1679 + /** Set when the provider did not confirm the stop in time; clients then offer Force stop. */
1680 + unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtc),
1681 }),
1682 Schema.Struct({
1683 ...OrchestrationV2TurnItemBaseFields,
@@ -2500,6 +2502,7 @@ export const OrchestrationV2TurnItemJson = Schema.Union([
2502 ...OrchestrationV2TurnItemJsonBaseFields,
2503 type: Schema.Literal("run_interrupt_request"),
2504 message: Schema.String,
2505 + unresponsiveAt: Schema.optionalKey(Schema.DateTimeUtcFromString),
2506 }),
2507 Schema.Struct({
2508 ...OrchestrationV2TurnItemJsonBaseFields,
@@ -3436,12 +3439,22 @@ const OrchestrationV2InternalCommand = Schema.Union([
3439 threadId: ThreadId,
3440 requestId: RuntimeRequestId,
3441 }),
3442 + /** The provider did not confirm a Stop of this turn in time. */
3443 + Schema.Struct({
3444 + type: Schema.Literal("thread.interrupt.unresponsive"),
3445 + commandId: CommandId,
3446 + threadId: ThreadId,
3447 + providerThreadId: ProviderThreadId,
3448 + providerTurnId: ProviderTurnId,
3449 + }),
3450 Schema.Struct({
3451 type: Schema.Literal("thread.background-work.settle"),
3452 commandId: CommandId,
3453 threadId: ThreadId,
3454 providerThreadId: ProviderThreadId,
3455 providerTurnId: ProviderTurnId,
3456 + /** Replaces the Stop's own reason on the result, as Force stop does. */
3457 + reason: Schema.optional(Schema.String),
3458 subagent: Schema.optional(
3459 Schema.Struct({
3460 id: NodeId,
@@ -3521,6 +3534,7 @@ export const ORCHESTRATION_V2_WS_METHODS = {
3534 getThreadProjection: "orchestration.getThreadProjection",
3535 getWorkflowScript: "orchestration.getWorkflowScript",
3536 stopWorkflow: "orchestration.stopWorkflow",
3537 + forceStopThread: "orchestration.forceStopThread",
3538 getTurnItem: "orchestration.getTurnItem",
3539 launchThread: "orchestration.launchThread",
3540 subscribeShell: "orchestration.subscribeShell",
@@ -3908,6 +3922,41 @@ export class OrchestrationV2StopWorkflowError extends Schema.TaggedError<Orchest
3922 }
3923 }
3924
3925 +export const OrchestrationV2ForceStopThreadInput = Schema.Struct({
3926 + threadId: ThreadId,
3927 + /** The run the user saw ignore Stop; Force stop refuses once a newer run has started. */
3928 + runId: Schema.optionalKey(RunId),
3929 +});
3930 +export type OrchestrationV2ForceStopThreadInput = typeof OrchestrationV2ForceStopThreadInput.Type;
3931 +
3932 +export const OrchestrationV2ForceStopThreadResult = Schema.Struct({
3933 + /**
3934 + * `killed`: the thread's own agent processes were killed. `released`: its provider session
3935 + * was torn down, which kills the agent's process group. `releasing`: that teardown has not
3936 + * finished yet and continues on the server. `shared`: the agent process also works for
3937 + * `sharedWithThreadIds`, so only this thread's turn was ended.
3938 + */
3939 + outcome: Schema.Literals(["killed", "released", "releasing", "shared"]),
3940 + sharedWithThreadIds: Schema.Array(ThreadId),
3941 +});
3942 +export type OrchestrationV2ForceStopThreadResult = typeof OrchestrationV2ForceStopThreadResult.Type;
3943 +
3944 +export class OrchestrationV2ForceStopThreadError extends Schema.TaggedError<OrchestrationV2ForceStopThreadError>()(
3945 + "OrchestrationV2ForceStopThreadError",
3946 + {
3947 + threadId: ThreadId,
3948 + /** `superseded`: a newer run started since the user asked; nothing was stopped. */
3949 + reason: Schema.optional(Schema.Literal("superseded")),
3950 + cause: Schema.optional(Schema.Defect()),
3951 + },
3952 +) {
3953 + override get message(): string {
3954 + return this.reason === "superseded"
3955 + ? "A new turn started, so nothing was force stopped."
3956 + : "Failed to force stop the agent.";
3957 + }
3958 +}
3959 +
3960 export const OrchestrationV2GetWorkflowScriptInput = Schema.Struct({
3961 threadId: ThreadId,
3962 scriptPath: TrimmedNonEmptyString,
@@ -4094,6 +4143,10 @@ export const OrchestrationV2RpcSchemas = {
4143 input: OrchestrationV2StopWorkflowInput,
4144 output: Schema.Void,
4145 },
4146 + forceStopThread: {
4147 + input: OrchestrationV2ForceStopThreadInput,
4148 + output: OrchestrationV2ForceStopThreadResult,
4149 + },
4150 getWorkflowScript: {
4151 input: OrchestrationV2GetWorkflowScriptInput,
4152 output: OrchestrationV2GetWorkflowScriptResult,
packages/contracts/src/rpc.ts
+8
@@ -220,6 +220,7 @@ import {
220 ORCHESTRATION_V2_WS_METHODS,
221 OrchestrationGetWorkflowScriptError,
222 OrchestrationV2StopWorkflowError,
223 + OrchestrationV2ForceStopThreadError,
224 OrchestrationV2DispatchCommandError,
225 OrchestrationV2GetShellSnapshotError,
226 OrchestrationV2CommandOutputError,
@@ -1770,6 +1771,12 @@ const WsOrchestrationV2StopWorkflowRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.st
1771 error: Schema.Union([OrchestrationV2StopWorkflowError, EnvironmentAuthorizationError]),
1772 });
1773
1774 +const WsOrchestrationV2ForceStopThreadRpc = Rpc.make(ORCHESTRATION_V2_WS_METHODS.forceStopThread, {
1775 + payload: OrchestrationV2RpcSchemas.forceStopThread.input,
1776 + success: OrchestrationV2RpcSchemas.forceStopThread.output,
1777 + error: Schema.Union([OrchestrationV2ForceStopThreadError, EnvironmentAuthorizationError]),
1778 +});
1779 +
1780 const WsOrchestrationV2GetWorkflowScriptRpc = Rpc.make(
1781 ORCHESTRATION_V2_WS_METHODS.getWorkflowScript,
1782 {
@@ -2122,6 +2129,7 @@ export const WsRpcGroup = RpcGroup.make(
2129 WsOrchestrationV2DispatchCommandRpc,
2130 WsOrchestrationV2GetWorkflowScriptRpc,
2131 WsOrchestrationV2StopWorkflowRpc,
2132 + WsOrchestrationV2ForceStopThreadRpc,
2133 WsOrchestrationV2GetTurnItemRpc,
2134 WsOrchestrationV2GetTurnDiffRpc,
2135 WsOrchestrationV2GetFullThreadDiffRpc,
packages/provider-acp/src/server/adapter.ts
+40 -21
@@ -1324,6 +1324,8 @@ interface SnapshotMessageState {
1324 loadingIndex: number;
1325 }
1326
1327 +/** How long a Stop waits for the agent to confirm before the user is offered Force stop. */
1328 +const ACP_STOP_CONFIRM_TIMEOUT_MS = 10_000;
1329 const TOOL_UPDATE_PERSIST_EVERY = 10;
1330
1331 // Agents can resend whole calls while streaming arguments.
@@ -7989,7 +7991,9 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
7991 return yield* Effect.uninterruptibleMask((restore) =>
7992 Effect.gen(function* () {
7993 yield* restore(runtimeTransitionPermit.take(1));
7992 - return yield* Effect.gen(function* () {
7994 + // Stop's caller times out on this call. The transition runs in its own fiber so
7995 + // a stalled hard teardown keeps its state and permit without holding the caller.
7996 + const transitionFiber = yield* Effect.gen(function* () {
7997 const transition = yield* Effect.gen(function* () {
7998 const interruptContext = yield* Ref.get(activeTurn);
7999 if (
@@ -8000,8 +8004,12 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8004 ) {
8005 // ACP cancel leaves native children alive; close confirms session-wide Stop.
8006 pendingStopCloseTurn = interruptContext;
8003 - const closed = yield* runtime.closeSession().pipe(
8004 - Effect.timeoutOption("10 seconds"),
8007 + // Interruptible, or neither this timeout nor the caller's could end a hung close.
8008 + const closed = yield* restore(
8009 + runtime
8010 + .closeSession()
8011 + .pipe(Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS)),
8012 + ).pipe(
8013 Effect.mapError(
8014 (cause) =>
8015 new ProviderAdapter.ProviderAdapterProtocolError({
@@ -8013,10 +8021,11 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8021 ),
8022 );
8023 if (Option.isNone(closed)) {
8016 - return yield* new ProviderAdapter.ProviderAdapterProtocolError({
8024 + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
8025 driver,
8018 - detail:
8019 - "The agent did not confirm session close; background work may still run",
8026 + providerThreadId: turnInput.providerThread.id,
8027 + providerTurnId: turnInput.providerTurnId,
8028 + timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS,
8029 });
8030 }
8031 yield* runtimeCallbackPermit.withPermit(
@@ -8246,15 +8255,16 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8255 }
8256 }
8257 const stopped = yield* Deferred.await(context.completed).pipe(
8249 - Effect.timeoutOption("10 seconds"),
8258 + Effect.timeoutOption(ACP_STOP_CONFIRM_TIMEOUT_MS),
8259 );
8260 + // The turn stays open: the agent may still be working, and the user
8261 + // decides whether to Force stop it.
8262 if (Option.isNone(stopped)) {
8252 - if (!context.finalized) {
8253 - yield* finalizeTurn(context, "interrupted");
8254 - }
8255 - return yield* new ProviderAdapter.ProviderAdapterProtocolError({
8263 + return yield* new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
8264 driver,
8257 - detail: `ACP provider turn ${turnInput.providerTurnId} did not acknowledge cancellation before the interrupt timeout`,
8265 + providerThreadId: turnInput.providerThread.id,
8266 + providerTurnId: turnInput.providerTurnId,
8267 + timeoutMs: ACP_STOP_CONFIRM_TIMEOUT_MS,
8268 });
8269 }
8270 });
@@ -8275,20 +8285,29 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
8285 }),
8286 ),
8287 );
8278 - }).pipe(Effect.ensuring(runtimeTransitionPermit.release(1)));
8288 + }).pipe(
8289 + Effect.ensuring(runtimeTransitionPermit.release(1)),
8290 + Effect.forkIn(sessionScope, { startImmediately: true, uninterruptible: true }),
8291 + );
8292 + return yield* restore(Fiber.join(transitionFiber)).pipe(
8293 + Effect.onInterrupt(() =>
8294 + Fiber.interrupt(transitionFiber).pipe(Effect.forkDetach, Effect.asVoid),
8295 + ),
8296 + );
8297 }),
8298 );
8299 },
8300 (effect, turnInput) =>
8301 effect.pipe(
8284 - Effect.mapError(
8285 - (cause) =>
8286 - new ProviderAdapter.ProviderAdapterInterruptError({
8287 - driver,
8288 - providerThreadId: turnInput.providerThread.id,
8289 - providerTurnId: turnInput.providerTurnId,
8290 - cause,
8291 - }),
8302 + Effect.mapError((cause) =>
8303 + cause._tag === "ProviderAdapterInterruptTimeoutError"
8304 + ? cause
8305 + : new ProviderAdapter.ProviderAdapterInterruptError({
8306 + driver,
8307 + providerThreadId: turnInput.providerThread.id,
8308 + providerTurnId: turnInput.providerTurnId,
8309 + cause,
8310 + }),
8311 ),
8312 ),
8313 ),
packages/provider-core/src/server/ProviderAdapter.ts
+16
@@ -327,6 +327,21 @@ export class ProviderAdapterInterruptError extends Schema.TaggedError<ProviderAd
327 }
328 }
329
330 +/** The provider did not confirm a stop in time; its processes may still be running. */
331 +export class ProviderAdapterInterruptTimeoutError extends Schema.TaggedError<ProviderAdapterInterruptTimeoutError>()(
332 + "ProviderAdapterInterruptTimeoutError",
333 + {
334 + driver: ProviderDriverKind,
335 + providerThreadId: ProviderThreadId,
336 + providerTurnId: ProviderTurnId,
337 + timeoutMs: Schema.Number,
338 + },
339 +) {
340 + override get message(): string {
341 + return `The ${this.driver} provider did not confirm stopping turn ${this.providerTurnId} within ${this.timeoutMs} ms.`;
342 + }
343 +}
344 +
345 export class ProviderAdapterRuntimeRequestResponseError extends Schema.TaggedError<ProviderAdapterRuntimeRequestResponseError>()(
346 "ProviderAdapterRuntimeRequestResponseError",
347 {
@@ -388,6 +403,7 @@ export const ProviderAdapterV2Error = Schema.Union([
403 ProviderAdapterSteerRunUnsupportedError,
404 ProviderAdapterSteerRunError,
405 ProviderAdapterInterruptError,
406 + ProviderAdapterInterruptTimeoutError,
407 ProviderAdapterRuntimeRequestResponseError,
408 ProviderAdapterEventStreamError,
409 ProviderAdapterProtocolError,
packages/shared/src/AgentScope.ts
+7
@@ -22,6 +22,12 @@ export interface AgentScopeShape {
22 * descendants outside its process group. Preserves systemd OOM diagnosis.
23 */
24 readonly stop: (unit: string) => Effect.Effect<void>;
25 + /**
26 + * SIGKILLs every process in the thread's own scope, which works even while
27 + * the kernel throttles them. Shared scopes serve other threads and are never
28 + * killed here. Succeeds with whether a scope was killed.
29 + */
30 + readonly kill: (threadId: string) => Effect.Effect<boolean>;
31 /**
32 * Forgets the thread's scope. The session manager calls it before it opens
33 * a provider session, so a scope only answers for the session that launched
@@ -42,6 +48,7 @@ export const AgentScope = Context.Reference<AgentScopeShape>("@t3tools/shared/Ag
48 wrap: ({ command, args }) => Effect.succeed({ command, args }),
49 oomKilled: () => Effect.succeed(false),
50 stop: () => Effect.void,
51 + kill: () => Effect.succeed(false),
52 clear: () => Effect.void,
53 }),
54 });