@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 3,519 lines 151 KB
Raw
1 import { AgentScope } from "@t3tools/shared/AgentScope";
2 import * as KeyedLock from "@t3tools/shared/KeyedLock";
3 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
4 import {
5 isOrchestrationV2WorkActive,
6 ModelSelection,
7 type OrchestrationV2AppThread,
8 OrchestrationV2DomainEvent,
9 OrchestrationV2ProviderSession,
10 type OrchestrationV2PromptSuggestion,
11 type OrchestrationV2ProviderThread,
12 type OrchestrationV2Run,
13 OrchestrationV2RuntimeRequest,
14 type OrchestrationV2Subagent,
15 ProviderInstanceId,
16 ProviderSessionId,
17 ThreadId,
18 type OrchestrationV2TurnItem,
19 type ProviderThreadId,
20 type ProviderTurnId,
21 } from "@t3tools/contracts";
22 import * as Cause from "effect/Cause";
23 import * as Clock from "effect/Clock";
24 import * as Context from "effect/Context";
25 import * as DateTime from "effect/DateTime";
26 import * as Deferred from "effect/Deferred";
27 import * as Duration from "effect/Duration";
28 import * as Effect from "effect/Effect";
29 import * as Exit from "effect/Exit";
30 import * as Fiber from "effect/Fiber";
31 import * as FiberSet from "effect/FiberSet";
32 import * as FileSystem from "effect/FileSystem";
33 import * as Layer from "effect/Layer";
34 import * as Option from "effect/Option";
35 import * as Queue from "effect/Queue";
36 import * as Ref from "effect/Ref";
37 import * as Schedule from "effect/Schedule";
38 import * as Schema from "effect/Schema";
39 import * as Scope from "effect/Scope";
40 import * as Semaphore from "effect/Semaphore";
41 import * as Stream from "effect/Stream";
42 import * as SubscriptionRef from "effect/SubscriptionRef";
43
44 import { normalizeModelMetricLabel } from "../observability/Attributes.ts";
45 import {
46 providerSessionsTotal,
47 providerTurnDuration,
48 providerTurnsTotal,
49 withMetrics,
50 } from "../observability/Metrics.ts";
51 import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
52 import * as ProviderInstallationGate from "../provider/ProviderInstallationGate.ts";
53 import * as ProjectService from "../project/ProjectService.ts";
54 import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
55 import * as ServerSettings from "../serverSettings.ts";
56 import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
57 import * as EventSink from "./EventSink.ts";
58 import * as CommandOutputHub from "./CommandOutputHub.ts";
59 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
60 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
61 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
62 import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
63 import {
64 endOrphanedNativeSubagent,
65 endRunlessRootTurns,
66 } from "@t3tools/provider-core/server/subagentProjection";
67 import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
68 import * as ProjectionStore from "./ProjectionStore.ts";
69
70 const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000;
71 const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000;
72 const RELEASE_SCOPE_CLOSE_TIMEOUT_MS = 30 * 1000;
73 /** Past adapters' own 10 s stop fallbacks, so they settle first when the provider answers at all. */
74 export const PROVIDER_INTERRUPT_TIMEOUT_MS = 15 * 1000;
75
76 const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThreadId}#`;
77 const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) =>
78 `${busyTurnPrefix(providerThreadId)}${runOrdinal}`;
79 interface OwnBusyTurns {
80 readonly providerThreadIds: ReadonlyArray<ProviderThreadId>;
81 readonly throughRunOrdinal: number;
82 }
83 const isOwnBusyTurn = (turnKey: string, own: OwnBusyTurns) =>
84 own.providerThreadIds.some((id) => {
85 const prefix = busyTurnPrefix(id);
86 return (
87 turnKey.startsWith(prefix) && Number(turnKey.slice(prefix.length)) <= own.throughRunOrdinal
88 );
89 });
90 const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000;
91 // Providers can announce a retry and never resume. Longer than any adapter's own stall handling.
92 const PROVIDER_RETRY_STALL_TIMEOUT = Duration.minutes(15);
93 type ProviderErrorItem = Extract<OrchestrationV2TurnItem, { readonly type: "error" }>;
94 type ProviderRetryItem = ProviderErrorItem & {
95 readonly providerTurnId: ProviderTurnId;
96 readonly providerThreadId: ProviderThreadId;
97 readonly retry: NonNullable<ProviderErrorItem["retry"]>;
98 };
99 /** Suggestions outlive their sessions, so keep only the most recently updated threads. */
100 const MAX_PROMPT_SUGGESTIONS = 256;
101 const WORKING_RUN_STATUSES: ReadonlySet<string> = new Set([
102 "preparing",
103 "starting",
104 "running",
105 "waiting",
106 ]);
107
108 /** Matches planDelegatedCompletionDelivery: true while a delegated task's result still wakes the thread. */
109 export const delegatedTaskWillWake = (records: {
110 readonly thread: Pick<OrchestrationV2AppThread, "archivedAt" | "deletedAt">;
111 readonly runs: ReadonlyArray<Pick<OrchestrationV2Run, "id" | "delegatedCompletion">>;
112 readonly subagents: ReadonlyArray<
113 Pick<OrchestrationV2Subagent, "origin" | "status" | "runId" | "completionDelivery">
114 >;
115 }): boolean =>
116 records.thread.archivedAt === null &&
117 records.thread.deletedAt === null &&
118 records.subagents.some((task) => {
119 const run = records.runs.find((candidate) => candidate.id === task.runId);
120 const delivery = task.completionDelivery?.state;
121 return (
122 task.origin === "app_owned" &&
123 isOrchestrationV2WorkActive(task.status) &&
124 run !== undefined &&
125 (run.delegatedCompletion?.disposition ?? "open") === "open" &&
126 delivery !== "acknowledged" &&
127 delivery !== "delivered" &&
128 delivery !== "disposed"
129 );
130 });
131
132 export const ProviderSessionReleaseReason = Schema.Literals([
133 "idle_timeout",
134 "runtime_error",
135 "manual_shutdown",
136 "server_shutdown",
137 ]);
138 export type ProviderSessionReleaseReason = typeof ProviderSessionReleaseReason.Type;
139
140 // Tool calls that only a live provider process can end.
141 const releasedToolCallTypes: ReadonlyArray<OrchestrationV2TurnItem["type"]> = [
142 "command_execution",
143 "dynamic_tool",
144 ];
145 const activeTurnItemStatuses: ReadonlyArray<OrchestrationV2TurnItem["status"]> = [
146 "pending",
147 "running",
148 "waiting",
149 ];
150
151 export class ProviderSessionOpenError extends Schema.TaggedError<ProviderSessionOpenError>()(
152 "ProviderSessionOpenError",
153 {
154 instanceId: ProviderInstanceId,
155 providerSessionId: ProviderSessionId,
156 cause: Schema.optional(Schema.Defect()),
157 },
158 ) {
159 override get message(): string {
160 return `Failed to open provider instance ${this.instanceId} session ${this.providerSessionId}.`;
161 }
162 }
163
164 export class ProviderSessionLookupError extends Schema.TaggedError<ProviderSessionLookupError>()(
165 "ProviderSessionLookupError",
166 {
167 providerSessionId: ProviderSessionId,
168 cause: Schema.optional(Schema.Defect()),
169 },
170 ) {
171 override get message(): string {
172 return `Failed to look up provider session ${this.providerSessionId}.`;
173 }
174 }
175
176 export class ProviderSessionCloseError extends Schema.TaggedError<ProviderSessionCloseError>()(
177 "ProviderSessionCloseError",
178 {
179 providerSessionId: ProviderSessionId,
180 cause: Schema.optional(Schema.Defect()),
181 },
182 ) {
183 override get message(): string {
184 return `Failed to close provider session ${this.providerSessionId}.`;
185 }
186 }
187
188 export class ProviderSessionReleaseError extends Schema.TaggedError<ProviderSessionReleaseError>()(
189 "ProviderSessionReleaseError",
190 {
191 providerSessionId: ProviderSessionId,
192 reason: ProviderSessionReleaseReason,
193 cause: Schema.optional(Schema.Defect()),
194 },
195 ) {
196 override get message(): string {
197 return `Failed to release provider session ${this.providerSessionId}.`;
198 }
199 }
200
201 export class ProviderSessionActivityError extends Schema.TaggedError<ProviderSessionActivityError>()(
202 "ProviderSessionActivityError",
203 {
204 providerSessionId: ProviderSessionId,
205 cause: Schema.optional(Schema.Defect()),
206 },
207 ) {
208 override get message(): string {
209 return `Failed to update provider session activity for ${this.providerSessionId}.`;
210 }
211 }
212
213 export const ProviderSessionManagerV2Error = Schema.Union([
214 ProviderSessionOpenError,
215 ProviderWorkspaceMissingError,
216 ProviderSessionLookupError,
217 ProviderSessionCloseError,
218 ProviderSessionReleaseError,
219 ProviderSessionActivityError,
220 ]);
221 export type ProviderSessionManagerV2Error = typeof ProviderSessionManagerV2Error.Type;
222
223 export interface ProviderSessionManagerV2Shape {
224 readonly shutdown: Effect.Effect<void>;
225 readonly open: (input: {
226 readonly threadId: ThreadId;
227 readonly providerSessionId: ProviderSessionId;
228 readonly modelSelection: ModelSelection;
229 readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
230 readonly resumeFromSession?: OrchestrationV2ProviderSession;
231 readonly initialNativeThreadId?: string;
232 readonly initialProviderItemIdentityVersion?: 2;
233 }) => Effect.Effect<
234 ProviderAdapter.ProviderAdapterV2SessionRuntime,
235 ProviderSessionManagerV2Error
236 >;
237 readonly get: (
238 providerSessionId: ProviderSessionId,
239 ) => Effect.Effect<
240 Option.Option<ProviderAdapter.ProviderAdapterV2SessionRuntime>,
241 ProviderSessionManagerV2Error
242 >;
243 readonly isLive: (providerSessionId: ProviderSessionId) => Effect.Effect<boolean>;
244 readonly close: (
245 providerSessionId: ProviderSessionId,
246 ) => Effect.Effect<void, ProviderSessionManagerV2Error>;
247 readonly closeInstance: (
248 instanceId: ProviderInstanceId,
249 ) => Effect.Effect<void, ProviderSessionManagerV2Error>;
250 readonly release: (input: {
251 readonly providerSessionId: ProviderSessionId;
252 readonly reason: ProviderSessionReleaseReason;
253 readonly detail?: string;
254 }) => Effect.Effect<void, ProviderSessionManagerV2Error>;
255 readonly detach: (input: {
256 readonly providerSessionId: ProviderSessionId;
257 readonly threadId: ThreadId;
258 readonly detail?: string;
259 readonly revokeMcpCredential?: boolean;
260 }) => Effect.Effect<void, ProviderSessionManagerV2Error>;
261 /**
262 * Releases the session unless, at the moment of removal, a turn is busy on it other than one
263 * of `ownProviderThreadIds` from a run up to `throughRunOrdinal`. Checked atomically with turn
264 * starts. Succeeds with whether it was released.
265 */
266 readonly releaseUnlessBusy: (input: {
267 readonly providerSessionId: ProviderSessionId;
268 readonly ownProviderThreadIds: ReadonlyArray<ProviderThreadId>;
269 readonly throughRunOrdinal: number;
270 /** Keeps the session when it succeeds with true, checked atomically with the removal. */
271 readonly keepIf?: Effect.Effect<boolean>;
272 readonly reason: ProviderSessionReleaseReason;
273 readonly detail?: string;
274 }) => Effect.Effect<boolean, ProviderSessionManagerV2Error>;
275 /**
276 * Other threads with a run or background work on this session's shared process; empty when
277 * it serves one thread.
278 */
279 readonly otherWorkingThreads: (input: {
280 readonly providerSessionId: ProviderSessionId;
281 readonly threadId: ThreadId;
282 }) => Effect.Effect<ReadonlyArray<ThreadId>>;
283 /**
284 * Forgets that the thread has this provider thread loaded, so its next resume reaches the
285 * adapter. Leaves the attachment, idle unload and MCP credential alone, unlike detach.
286 */
287 readonly forgetLoadedProviderThread: (input: {
288 readonly providerSessionId: ProviderSessionId;
289 readonly threadId: ThreadId;
290 readonly providerThread: OrchestrationV2ProviderThread;
291 readonly modelSelection: ModelSelection;
292 readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
293 }) => Effect.Effect<void>;
294 /** The thread's latest predicted next prompt, then each replacement. */
295 readonly subscribePromptSuggestion: (
296 threadId: ThreadId,
297 ) => Stream.Stream<OrchestrationV2PromptSuggestion>;
298 }
299
300 export class ProviderSessionManagerV2 extends Context.Service<
301 ProviderSessionManagerV2,
302 ProviderSessionManagerV2Shape
303 >()("t3/orchestration-v2/ProviderSessionManager/ProviderSessionManagerV2") {}
304
305 interface LiveSessionEntry {
306 readonly attachedThreadIds: ReadonlySet<ThreadId>;
307 readonly loadedProviderThreadKeyByThread: ReadonlyMap<ThreadId, string>;
308 /** null marks an unknown catalog; undefined means MCP refresh is not configured. */
309 readonly openedWithLinkedEnvironments: boolean | null | undefined;
310 readonly linkedEnvironmentsByThread: ReadonlyMap<ThreadId, boolean | null>;
311 /** Scope revocation to recorded credential ids so superseded sessions cannot revoke replacements opened before detach. */
312 readonly mcpCredentialIdByThread: ReadonlyMap<ThreadId, string>;
313 readonly supportsMultipleProviderThreads: boolean;
314 readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
315 readonly exposedRuntime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
316 readonly eventSubscribers: Ref.Ref<
317 ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
318 >;
319 readonly requestEventPermit: Semaphore.Semaphore;
320 readonly scope: Scope.Closeable;
321 readonly idleGeneration: number;
322 readonly busyTurns: ReadonlySet<string>;
323 readonly idleFiber: Fiber.Fiber<void, never> | null;
324 readonly pinnedSinceMs: number | null;
325 readonly idleThreadUnloads: ReadonlyMap<ThreadId, IdleThreadUnload>;
326 }
327
328 interface IdleThreadUnload {
329 readonly providerThread: OrchestrationV2ProviderThread;
330 readonly generation: number;
331 readonly fiber: Fiber.Fiber<void, never> | null;
332 }
333
334 type ProviderSessionEventSignal =
335 | { readonly type: "event"; readonly event: ProviderAdapter.ProviderAdapterV2Event }
336 | {
337 readonly type: "failure";
338 readonly cause: Cause.Cause<ProviderAdapter.ProviderAdapterV2Error>;
339 };
340
341 export interface ProviderSessionManagerV2LayerOptions {
342 readonly idleTimeoutMs?: number;
343 readonly maxIdlePinMs?: number;
344 readonly configureMcp?: boolean;
345 }
346
347 function releaseStatusFor(
348 reason: ProviderSessionReleaseReason,
349 ): OrchestrationV2ProviderSession["status"] {
350 return reason === "runtime_error" ? "error" : "stopped";
351 }
352
353 function releasedRuntimeRequestStatusFor(
354 reason: ProviderSessionReleaseReason,
355 ): OrchestrationV2RuntimeRequest["status"] {
356 return reason === "manual_shutdown" || reason === "server_shutdown" ? "cancelled" : "expired";
357 }
358
359 function sessionKey(providerSessionId: ProviderSessionId): string {
360 return String(providerSessionId);
361 }
362
363 function sessionScopedArtifactThreadId(
364 event: ProviderAdapter.ProviderAdapterV2Event,
365 ): ThreadId | undefined {
366 switch (event.type) {
367 case "provider_thread.updated":
368 return event.sessionScoped === true
369 ? (event.providerThread.appThreadId ?? undefined)
370 : undefined;
371 case "runtime_request.updated":
372 return event.runtimeRequest.providerTurnId === null ? event.threadId : undefined;
373 case "node.updated":
374 return event.node.runId === null && event.node.runtimeRequestId !== null
375 ? event.node.threadId
376 : undefined;
377 case "turn_item.updated":
378 if (event.turnItem.type === "command_execution")
379 return event.sessionScoped === true ? event.turnItem.threadId : undefined;
380 return event.turnItem.runId === null &&
381 (event.turnItem.type === "approval_request" || event.turnItem.type === "user_input_request")
382 ? event.turnItem.threadId
383 : undefined;
384 default:
385 return undefined;
386 }
387 }
388
389 function providerThreadRuntimeKey(
390 providerThread: Parameters<
391 ProviderAdapter.ProviderAdapterV2SessionRuntime["resumeThread"]
392 >[0]["providerThread"],
393 ): string {
394 const nativeThreadRef = providerThread.nativeThreadRef;
395 return nativeThreadRef === null
396 ? String(providerThread.id)
397 : `${nativeThreadRef.driver}:${nativeThreadRef.nativeId}`;
398 }
399
400 function providerThreadLoadKey(input: {
401 readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
402 readonly providerThread: Parameters<
403 ProviderAdapter.ProviderAdapterV2SessionRuntime["resumeThread"]
404 >[0]["providerThread"];
405 readonly modelSelection?: ModelSelection;
406 readonly runtimePolicy?: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
407 }): string {
408 const modelSelection =
409 input.modelSelection === undefined
410 ? undefined
411 : (input.runtime.normalizeThreadLoadModelSelection?.(input.modelSelection) ??
412 input.modelSelection);
413 return JSON.stringify({
414 providerThread: providerThreadRuntimeKey(input.providerThread),
415 modelSelection: modelSelection ?? null,
416 runtimePolicy: input.runtimePolicy ?? null,
417 });
418 }
419
420 export const layerWithOptions = (
421 options: ProviderSessionManagerV2LayerOptions = {},
422 ): Layer.Layer<
423 ProviderSessionManagerV2,
424 never,
425 | CommandOutputHub.CommandOutputHub
426 | EventSink.EventSinkV2
427 | FileSystem.FileSystem
428 | IdAllocator.IdAllocatorV2
429 | McpProviderSessions.McpProviderSessions
430 | McpSessionRegistry.McpSessionRegistry
431 | ProjectionStore.ProjectionStoreV2
432 | ProviderEventIngestor.ProviderEventIngestorV2
433 | ProviderAdapterRegistry.ProviderAdapterRegistryV2
434 | ProviderInstallationGate.ProviderInstallationGate
435 > =>
436 Layer.effect(
437 ProviderSessionManagerV2,
438 Effect.gen(function* () {
439 const registry = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2;
440 const fileSystem = yield* FileSystem.FileSystem;
441 const mcpSessionRegistry = yield* McpSessionRegistry.McpSessionRegistry;
442 const mcpSessions = yield* McpProviderSessions.McpProviderSessions;
443 /** Unreadable settings must preserve explicit browser-access opt-outs. */
444 const serverSettings = yield* Effect.serviceOption(ServerSettings.ServerSettingsService);
445 const projectService = yield* Effect.serviceOption(ProjectService.ProjectService);
446 const eventSink = yield* EventSink.EventSinkV2;
447 const commandOutput = yield* CommandOutputHub.CommandOutputHub;
448 const promptSuggestionStore = yield* SubscriptionRef.make<
449 ReadonlyMap<ThreadId, NonNullable<OrchestrationV2PromptSuggestion>>
450 >(new Map());
451 const idAllocator = yield* IdAllocator.IdAllocatorV2;
452 const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
453 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
454 const agentScope = yield* AgentScope;
455 const installationGate = yield* ProviderInstallationGate.ProviderInstallationGate;
456 // Labs flag, read live: Codex per turn (its shared session outlives a toggle), Claude per query.
457 // Unreadable settings keep it off.
458 const promptSuggestionsEnabled = Option.isNone(serverSettings)
459 ? Effect.succeed(false)
460 : serverSettings.value.getSettings.pipe(
461 Effect.map((settings) => settings.promptSuggestionsEnabled),
462 Effect.orElseSucceed(() => false),
463 );
464 const agentAccessSettings = Effect.fn("ProviderSessionManagerV2.agentAccessSettings")(
465 function* (threadId: ThreadId) {
466 if (Option.isNone(serverSettings)) return { browser: true, device: false };
467 return yield* Effect.gen(function* () {
468 const settings = yield* serverSettings.value.getSettings;
469 const thread = yield* projectionStore.getThread(threadId);
470 const entries = Object.values(settings.projectSettingsOverrides);
471 const browserOverridden = entries.some(
472 (entry) => entry.enableAgentBrowserAccess !== undefined,
473 );
474 const deviceOverridden = entries.some(
475 (entry) => entry.enableAgentDeviceAccess !== undefined,
476 );
477 if (browserOverridden || deviceOverridden) {
478 const project = Option.isSome(projectService)
479 ? yield* projectService.value.getById(thread.projectId)
480 : Option.none();
481 if (Option.isNone(project))
482 return {
483 browser: browserOverridden ? false : settings.enableAgentBrowserAccess,
484 device:
485 settings.enableDeviceSupport &&
486 !deviceOverridden &&
487 settings.enableAgentDeviceAccess,
488 };
489 }
490 const effective = resolveProjectSettings(settings, thread.projectId).settings;
491 return {
492 browser: effective.enableAgentBrowserAccess,
493 device: settings.enableDeviceSupport && effective.enableAgentDeviceAccess,
494 };
495 }).pipe(
496 Effect.catch((cause) =>
497 Effect.logWarning(
498 "Could not resolve agent access; withholding browser and device tools.",
499 { threadId, cause },
500 ).pipe(Effect.as({ browser: false, device: false })),
501 ),
502 );
503 },
504 );
505 const readLinkedEnvironments = Effect.gen(function* () {
506 if (options.configureMcp === false || Option.isNone(serverSettings)) return undefined;
507 return yield* serverSettings.value.getSettings.pipe(
508 Effect.map((settings) => settings.linkedEnvironmentsEnabled),
509 Effect.orElseSucceed(() => null),
510 );
511 });
512 const layerScope = yield* Effect.scope;
513 // Group signals stop providers before restart capture; drop post-signal events so recovery owns that state.
514 const shutdownSignal = { received: false };
515 const onShutdownSignal = () => {
516 shutdownSignal.received = true;
517 };
518 yield* Effect.acquireRelease(
519 Effect.sync(() => {
520 process.on("SIGINT", onShutdownSignal);
521 process.on("SIGTERM", onShutdownSignal);
522 }),
523 () =>
524 Effect.sync(() => {
525 process.off("SIGINT", onShutdownSignal);
526 process.off("SIGTERM", onShutdownSignal);
527 }),
528 );
529 const sessions = yield* Ref.make(new Map<string, LiveSessionEntry>());
530 // Retry per released entry; reusing ids must not lose cleanup for earlier threads.
531 const releaseRecordRetries = yield* FiberSet.make();
532 const nextSubscriberId = yield* Ref.make(0);
533 const sessionOpen = yield* KeyedLock.make<ProviderSessionId>();
534 const threadAttachment = yield* KeyedLock.make<string>();
535 const threadAttachmentKey = (input: {
536 readonly providerSessionId: ProviderSessionId;
537 readonly threadId: ThreadId;
538 }) => `${input.providerSessionId}\u0000${input.threadId}`;
539 const idleTimeoutMs = Math.max(1, options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS);
540 const maxIdlePinMs = Math.max(0, options.maxIdlePinMs ?? DEFAULT_MAX_IDLE_PIN_MS);
541 interface PreparedMcpCredential {
542 readonly mcpCredentialId: string | undefined;
543 readonly issued: boolean;
544 }
545 /** ACP and OpenCode consume credentials during openSession; reserve them against revocation until publication. */
546 const mcpCredentialReservations = new Map<string, number>();
547 const mcpReservationKey = (threadId: ThreadId, mcpCredentialId: string) =>
548 `${threadId}\0${mcpCredentialId}`;
549 const reserveMcpCredential = (threadId: ThreadId, mcpCredentialId: string) => {
550 const key = mcpReservationKey(threadId, mcpCredentialId);
551 mcpCredentialReservations.set(key, (mcpCredentialReservations.get(key) ?? 0) + 1);
552 };
553 const dropMcpCredentialReservation = (threadId: ThreadId, mcpCredentialId: string) => {
554 const key = mcpReservationKey(threadId, mcpCredentialId);
555 const count = mcpCredentialReservations.get(key) ?? 0;
556 if (count <= 1) {
557 mcpCredentialReservations.delete(key);
558 } else {
559 mcpCredentialReservations.set(key, count - 1);
560 }
561 };
562 const isMcpCredentialReserved = (threadId: ThreadId, mcpCredentialId: string) =>
563 (mcpCredentialReservations.get(mcpReservationKey(threadId, mcpCredentialId)) ?? 0) > 0;
564 const mcpPrepareLock = yield* KeyedLock.make<ThreadId>();
565 /** Serialize prepares per thread so concurrent rotations cannot revoke each other's credentials. */
566 const prepareMcpSession = (
567 threadId: ThreadId,
568 providerInstanceId: ProviderInstanceId,
569 ): Effect.Effect<PreparedMcpCredential> =>
570 options.configureMcp === false
571 ? mcpSessions
572 .clear(threadId)
573 .pipe(Effect.as<PreparedMcpCredential>({ mcpCredentialId: undefined, issued: false }))
574 : mcpPrepareLock.withLock(
575 threadId,
576 Effect.gen(function* () {
577 // Live providers build MCP clients once; reuse credentials across workspace handoffs.
578 const { browser: browserToolsAvailable, device: deviceToolsAvailable } =
579 yield* agentAccessSettings(threadId);
580 const capabilities = new Set<
581 import("../mcp/McpInvocationContext.ts").McpCapability
582 >(["orchestration", "worktree", "pull-requests", "issues"]);
583 if (browserToolsAvailable) capabilities.add("preview");
584 if (deviceToolsAvailable) capabilities.add("device");
585 const backgroundWait = yield* registry.get(providerInstanceId).pipe(
586 Effect.flatMap((adapter) => adapter.getCapabilities()),
587 Effect.map((adapter) => adapter.tools.supportsBackgroundWait === true),
588 Effect.option,
589 );
590 if (Option.getOrElse(backgroundWait, () => false)) {
591 capabilities.add("background-wait");
592 }
593 const subagent = yield* projectionStore.getThread(threadId).pipe(
594 Effect.map((thread) => thread.lineage.relationshipToParent === "subagent"),
595 Effect.orElseSucceed(() => false),
596 );
597 const existing = yield* mcpSessions.read(threadId);
598 if (existing !== undefined) {
599 // Reserve before async validation so release cannot revoke between validation and reservation.
600 reserveMcpCredential(threadId, existing.providerSessionId);
601 const rawToken = existing.authorizationHeader.replace(/^Bearer\s+/, "");
602 // Interrupted resolution must release the reservation before the caller learns of it.
603 const resolved = yield* mcpSessionRegistry
604 .resolve(rawToken)
605 .pipe(
606 Effect.onInterrupt(() =>
607 Effect.sync(() =>
608 dropMcpCredentialReservation(threadId, existing.providerSessionId),
609 ),
610 ),
611 );
612 if (
613 resolved !== undefined &&
614 resolved.thread.threadId === threadId &&
615 resolved.thread.providerInstanceId === providerInstanceId &&
616 resolved.capabilities.has("preview") === browserToolsAvailable &&
617 resolved.capabilities.has("device") === deviceToolsAvailable &&
618 // A failed capability read cannot tell that the live credential is wrong.
619 (Option.isNone(backgroundWait) ||
620 resolved.capabilities.has("background-wait") === backgroundWait.value) &&
621 // A credential issued while the lineage read failed is replaced once it works.
622 (resolved.thread.subagent === true) === subagent
623 ) {
624 return { mcpCredentialId: existing.providerSessionId, issued: false };
625 }
626 dropMcpCredentialReservation(threadId, existing.providerSessionId);
627 }
628 yield* mcpSessionRegistry.revokeThread(threadId);
629 const credential = yield* mcpSessionRegistry.issue({
630 threadId,
631 providerInstanceId,
632 browserToolsAvailable,
633 capabilities,
634 subagent,
635 });
636 yield* mcpSessions.set({
637 ...credential.config,
638 hasPendingWake: projectionStore
639 .getThreadRecords(threadId, ["runs", "subagents"])
640 .pipe(
641 Effect.map(delegatedTaskWillWake),
642 Effect.orElseSucceed(() => false),
643 ),
644 });
645 reserveMcpCredential(threadId, credential.config.providerSessionId);
646 return { mcpCredentialId: credential.config.providerSessionId, issued: true };
647 }),
648 );
649 const clearMcpSession = (threadId: ThreadId, mcpCredentialId?: string) =>
650 mcpCredentialId === undefined
651 ? mcpSessionRegistry
652 .revokeThread(threadId)
653 .pipe(Effect.tap(() => mcpSessions.clear(threadId)))
654 : mcpSessionRegistry.revokeProviderSession(mcpCredentialId).pipe(
655 Effect.tap(() =>
656 Effect.gen(function* () {
657 if ((yield* mcpSessions.read(threadId))?.providerSessionId === mcpCredentialId) {
658 yield* mcpSessions.clear(threadId);
659 }
660 }),
661 ),
662 );
663
664 const subscriberReleaseCleanup = new WeakMap<
665 Queue.Queue<ProviderSessionEventSignal, Cause.Done>,
666 Effect.Effect<void>
667 >();
668
669 const publishToSubscribers = (
670 subscribers: Ref.Ref<
671 ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
672 >,
673 signal: ProviderSessionEventSignal,
674 ) =>
675 Ref.get(subscribers).pipe(
676 Effect.flatMap((current) =>
677 Effect.forEach(current.values(), (queue) => Queue.offer(queue, signal), {
678 discard: true,
679 }),
680 ),
681 );
682
683 const failSubscribers = Effect.fn("ProviderSessionManager.failSubscribers")(function* (
684 entry: LiveSessionEntry,
685 detail: string,
686 afterDrain: Effect.Effect<void>,
687 cause?: Cause.Cause<ProviderAdapter.ProviderAdapterV2Error>,
688 detachedThreadId?: ThreadId,
689 ) {
690 const error = new ProviderAdapter.ProviderAdapterEventStreamError({
691 driver: entry.runtime.driver,
692 providerSessionId: entry.runtime.providerSessionId,
693 raisedByT3: true,
694 ...(detachedThreadId === undefined ? {} : { detachedThreadId }),
695 cause: detail,
696 });
697 const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map());
698 yield* Effect.forEach(
699 subscribers.values(),
700 (queue) => {
701 subscriberReleaseCleanup.set(queue, afterDrain);
702 return Queue.offer(queue, {
703 type: "failure",
704 cause: cause ?? Cause.fail(error),
705 });
706 },
707 { discard: true },
708 );
709 });
710
711 // Server shutdown interrupts subscribers rather than ending them: a
712 // cleanly ended stream without a terminal fails its run, while an
713 // interrupted one leaves run settlement to restart recovery.
714 const interruptSubscribers = Effect.fn("ProviderSessionManager.interruptSubscribers")(
715 function* (entry: LiveSessionEntry) {
716 const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map());
717 yield* Effect.forEach(
718 subscribers.values(),
719 (queue) => Queue.clear(queue).pipe(Effect.andThen(Queue.interrupt(queue))),
720 { discard: true },
721 );
722 },
723 );
724
725 // Server shutdown intentionally clears them; a provider-announced Stop
726 // must let consumers drain them before the stream completes.
727 const endSubscribers = Effect.fn("ProviderSessionManager.endSubscribers")(function* (
728 entry: LiveSessionEntry,
729 afterDrain: Effect.Effect<void>,
730 ) {
731 const subscribers = yield* Ref.getAndSet(entry.eventSubscribers, new Map());
732 yield* Effect.forEach(
733 subscribers.values(),
734 (queue) => {
735 subscriberReleaseCleanup.set(queue, afterDrain);
736 return Queue.end(queue);
737 },
738 { discard: true },
739 );
740 });
741
742 const cancelIdleFiber = (fiber: Fiber.Fiber<void, never> | null) =>
743 fiber === null ? Effect.void : Fiber.interrupt(fiber).pipe(Effect.ignore);
744
745 const writeProviderSessionEvents = Effect.fn(
746 "ProviderSessionManager.writeProviderSessionEvents",
747 )(function* (input: {
748 readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
749 readonly threadIds: Iterable<ThreadId>;
750 readonly type: "provider-session.attached" | "provider-session.updated";
751 readonly payload: OrchestrationV2ProviderSession;
752 }) {
753 const now = yield* DateTime.now;
754 const events = yield* Effect.forEach(input.threadIds, (threadId) =>
755 Effect.gen(function* () {
756 return {
757 id: yield* idAllocator.allocate.event({
758 threadId,
759 providerSessionId: input.runtime.providerSessionId,
760 }),
761 type: input.type,
762 threadId,
763 driver: input.runtime.driver,
764 providerInstanceId: input.runtime.instanceId,
765 occurredAt: now,
766 payload: input.payload,
767 } satisfies OrchestrationV2DomainEvent;
768 }),
769 );
770 if (events.length > 0) {
771 yield* eventSink.write({ events });
772 }
773 });
774
775 const writeReleasedSessionEvents = Effect.fn(
776 "ProviderSessionManager.writeReleasedSessionEvents",
777 )(function* (input: {
778 readonly entry: LiveSessionEntry;
779 readonly reason: ProviderSessionReleaseReason;
780 readonly detail?: string;
781 }) {
782 const now = yield* DateTime.now;
783 const payload: OrchestrationV2ProviderSession = {
784 ...input.entry.runtime.providerSession,
785 status: releaseStatusFor(input.reason),
786 ...(input.entry.runtime.providerSession.extensionStatuses === undefined
787 ? {}
788 : { extensionStatuses: [] }),
789 updatedAt: now,
790 lastError:
791 input.reason === "runtime_error" ? (input.detail ?? "Provider runtime failed.") : null,
792 };
793 yield* writeProviderSessionEvents({
794 runtime: input.entry.runtime,
795 threadIds: input.entry.attachedThreadIds,
796 type: "provider-session.updated",
797 payload,
798 });
799 });
800
801 // A detach removes its thread before releasing, but that thread's records still need releasing.
802 const releasedThreadIds = (input: {
803 readonly entry: LiveSessionEntry;
804 readonly detachedThreadId?: ThreadId;
805 }) => {
806 const threadIds = new Set(input.entry.attachedThreadIds);
807 if (input.detachedThreadId !== undefined) threadIds.add(input.detachedThreadId);
808 return threadIds;
809 };
810
811 const writeReleasedRuntimeRequestEvents = Effect.fn(
812 "ProviderSessionManager.writeReleasedRuntimeRequestEvents",
813 )(function* (input: {
814 readonly entry: LiveSessionEntry;
815 readonly reason: ProviderSessionReleaseReason;
816 readonly releasedAt: DateTime.Utc;
817 readonly detachedThreadId?: ThreadId;
818 }) {
819 const providerSessionId = input.entry.runtime.providerSessionId;
820 const now = yield* DateTime.now;
821 const status = releasedRuntimeRequestStatusFor(input.reason);
822 const reason =
823 input.reason === "runtime_error"
824 ? "Provider session failed before this runtime request was resolved."
825 : "Provider session was closed before this runtime request was resolved.";
826
827 const events: Array<OrchestrationV2DomainEvent> = [];
828 for (const threadId of releasedThreadIds(input)) {
829 const projection = yield* projectionStore.getThreadRecords(
830 threadId,
831 ["runtimeRequests"],
832 { runtimeRequestStatuses: ["pending"] },
833 );
834 const releasedRequests = projection.runtimeRequests.filter(
835 (request) =>
836 request.status === "pending" &&
837 request.responseCapability.type === "live" &&
838 request.responseCapability.providerSessionId === providerSessionId &&
839 DateTime.isLessThanOrEqualTo(request.createdAt, input.releasedAt) &&
840 // Shutdown reconciliation decides which questions can survive the lost callback.
841 !(input.reason === "server_shutdown" && request.kind === "user_input"),
842 );
843
844 if (releasedRequests.length === 0) continue;
845 const records = yield* projectionStore.getThreadRecords(
846 threadId,
847 ["nodes", "turnItems"],
848 {
849 nodeIds: releasedRequests.map((request) => request.nodeId),
850 turnItemRequestIds: releasedRequests.map((request) => request.id),
851 turnItemTypes: ["approval_request", "user_input_request"],
852 },
853 );
854 for (const request of releasedRequests) {
855 events.push({
856 id: yield* idAllocator.allocate.event({
857 threadId,
858 providerSessionId,
859 }),
860 type: "runtime-request.updated",
861 threadId,
862 nodeId: request.nodeId,
863 driver: input.entry.runtime.driver,
864 occurredAt: now,
865 payload: {
866 ...request,
867 status,
868 responseCapability: {
869 type: "not_resumable",
870 reason,
871 },
872 resolvedAt: now,
873 },
874 });
875
876 const requestNode = records.nodes.find((node) => node.id === request.nodeId);
877 if (requestNode !== undefined) {
878 events.push({
879 id: yield* idAllocator.allocate.event({
880 threadId,
881 providerSessionId,
882 }),
883 type: "node.updated",
884 threadId,
885 ...(requestNode.runId === null ? {} : { runId: requestNode.runId }),
886 nodeId: requestNode.id,
887 driver: input.entry.runtime.driver,
888 occurredAt: now,
889 payload: {
890 ...requestNode,
891 status: input.reason === "runtime_error" ? "failed" : "cancelled",
892 completedAt: now,
893 },
894 });
895 }
896
897 const turnItem = records.turnItems.find(
898 (item) =>
899 (item.type === "approval_request" || item.type === "user_input_request") &&
900 item.requestId === request.id,
901 );
902 if (turnItem !== undefined) {
903 events.push({
904 id: yield* idAllocator.allocate.event({
905 threadId,
906 providerSessionId,
907 }),
908 type: "turn-item.updated",
909 threadId,
910 ...(turnItem.runId === null ? {} : { runId: turnItem.runId }),
911 ...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }),
912 driver: input.entry.runtime.driver,
913 occurredAt: now,
914 payload: {
915 ...turnItem,
916 status: input.reason === "runtime_error" ? "failed" : "cancelled",
917 completedAt: now,
918 updatedAt: now,
919 },
920 });
921 }
922 }
923 }
924
925 if (events.length > 0) {
926 yield* eventSink.write({ events });
927 }
928 });
929
930 // Read the session's provider threads as soon as it leaves the live map, before a
931 // replacement reports its own tasks. Keep OpenCode's background work: its background
932 // subagents share a server and can outlive the session.
933 const readReleasedProviderThreads = Effect.fn(
934 "ProviderSessionManager.readReleasedProviderThreads",
935 )(
936 function* (input: {
937 readonly entry: LiveSessionEntry;
938 readonly reason: ProviderSessionReleaseReason;
939 readonly detachedThreadId?: ThreadId;
940 }) {
941 if (
942 input.reason === "server_shutdown" ||
943 input.entry.runtime.backgroundTasksEndWithProcess !== true
944 ) {
945 return [];
946 }
947 const providerSessionId = input.entry.runtime.providerSessionId;
948 const released = [];
949 for (const threadId of releasedThreadIds(input)) {
950 const projection = yield* projectionStore.getThreadRecords(threadId, [
951 "providerThreads",
952 ]);
953 for (const providerThread of projection.providerThreads) {
954 if (providerThread.providerSessionId !== providerSessionId) continue;
955 released.push({
956 threadId,
957 providerSessionId,
958 providerThreadId: providerThread.id,
959 driver: providerThread.driver,
960 taskIds: new Set(
961 (providerThread.pendingBackgroundTasks ?? []).map((task) => task.taskId),
962 ),
963 });
964 }
965 }
966 return released;
967 },
968 (effect, input) =>
969 effect.pipe(
970 Effect.catchCause((cause) =>
971 Effect.logWarning("orchestration-v2.provider-session-release-roster-read-failed", {
972 providerSessionId: input.entry.runtime.providerSessionId,
973 cause,
974 }).pipe(Effect.as([])),
975 ),
976 ),
977 );
978 type ReleasedProviderThread = Effect.Success<
979 ReturnType<typeof readReleasedProviderThreads>
980 >[number];
981
982 // Dead processes cannot report task endings; stale rosters keep threads Waiting.
983 const writeReleasedRosterEvents = Effect.fn(
984 "ProviderSessionManager.writeReleasedRosterEvents",
985 )(function* ({
986 threadId,
987 providerSessionId,
988 providerThreadId,
989 taskIds,
990 }: ReleasedProviderThread) {
991 if (taskIds.size === 0) return;
992 for (let attempt = 0; attempt < 3; attempt++) {
993 const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"]);
994 const providerThread = projection.providerThreads.find(
995 (candidate) => candidate.id === providerThreadId,
996 );
997 const tasks = providerThread?.pendingBackgroundTasks ?? [];
998 const remaining = tasks.filter((task) => !taskIds.has(task.taskId));
999 if (providerThread === undefined || remaining.length === tasks.length) return;
1000 const now = yield* DateTime.now;
1001 const { committed } = yield* eventSink.writeIfProviderThreadUnchanged({
1002 expected: providerThread,
1003 events: [
1004 {
1005 id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
1006 type: "provider-thread.updated",
1007 threadId,
1008 driver: providerThread.driver,
1009 providerInstanceId: providerThread.providerInstanceId,
1010 occurredAt: now,
1011 payload: {
1012 ...providerThread,
1013 pendingBackgroundTasks: remaining,
1014 updatedAt: now,
1015 },
1016 },
1017 ],
1018 });
1019 if (committed) return;
1020 }
1021 yield* Effect.logWarning("orchestration-v2.provider-session-release-roster-conflict", {
1022 providerSessionId,
1023 providerThreadId,
1024 });
1025 });
1026
1027 // Tool calls the dead process started never report an end either, and run finalization
1028 // keeps them open for a late result. Items started after the release belong to a
1029 // replacement session.
1030 const writeReleasedToolCallEvents = Effect.fn(
1031 "ProviderSessionManager.writeReleasedToolCallEvents",
1032 )(function* (released: ReleasedProviderThread, releasedAt: DateTime.Utc) {
1033 const { threadId, providerSessionId, providerThreadId, driver } = released;
1034 const readCutOff = projectionStore
1035 .getThreadRecords(threadId, ["turnItems"], {
1036 turnItemTypes: releasedToolCallTypes,
1037 turnItemStatuses: activeTurnItemStatuses,
1038 })
1039 .pipe(
1040 Effect.map((projection) =>
1041 projection.turnItems.filter(
1042 (item) =>
1043 item.providerThreadId === providerThreadId &&
1044 DateTime.isLessThanOrEqualTo(item.startedAt ?? item.updatedAt, releasedAt),
1045 ),
1046 ),
1047 );
1048 // One write per item: a late completion from the dead session's queued events rejects
1049 // only the interrupt of its own item, and that item then is no longer open. Read again
1050 // after each pass: queued events can also store more cut-off items.
1051 let pending = yield* readCutOff;
1052 for (let pass = 0; pass < 3 && pending.length > 0; pass++) {
1053 const { nodes } = yield* projectionStore.getThreadRecords(threadId, ["nodes"], {
1054 nodeIds: pending.flatMap((item) => (item.nodeId === null ? [] : [item.nodeId])),
1055 });
1056 for (const item of pending) {
1057 const now = yield* DateTime.now;
1058 const node = nodes.find((candidate) => candidate.id === item.nodeId);
1059 const events: Array<OrchestrationV2DomainEvent> = [];
1060 if (node !== undefined && isOrchestrationV2WorkActive(node.status)) {
1061 events.push({
1062 id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
1063 type: "node.updated",
1064 threadId,
1065 ...(node.runId === null ? {} : { runId: node.runId }),
1066 nodeId: node.id,
1067 driver,
1068 occurredAt: now,
1069 payload: { ...node, status: "interrupted", completedAt: now },
1070 });
1071 }
1072 events.push({
1073 id: yield* idAllocator.allocate.event({ threadId, providerSessionId }),
1074 type: "turn-item.updated",
1075 threadId,
1076 ...(item.runId === null ? {} : { runId: item.runId }),
1077 ...(item.nodeId === null ? {} : { nodeId: item.nodeId }),
1078 driver,
1079 occurredAt: now,
1080 payload: { ...item, status: "interrupted", completedAt: now, updatedAt: now },
1081 });
1082 yield* eventSink.writeIfTurnItemUnchanged({ expected: item, events });
1083 }
1084 pending = yield* readCutOff;
1085 }
1086 if (pending.length === 0) return;
1087 yield* Effect.logWarning("orchestration-v2.provider-session-release-tool-call-conflict", {
1088 providerSessionId,
1089 providerThreadId,
1090 openToolCalls: pending.length,
1091 });
1092 });
1093
1094 // Background native subagents die with the process too, often after their run completed,
1095 // so run finalization does not end them. One started or resumed after the release belongs
1096 // to a replacement session.
1097 const writeReleasedSubagentEvents = Effect.fn(
1098 "ProviderSessionManager.writeReleasedSubagentEvents",
1099 )(function* (released: ReleasedProviderThread, releasedAt: DateTime.Utc) {
1100 const { threadId, providerSessionId, providerThreadId, driver } = released;
1101 const readCutOff = Effect.gen(function* () {
1102 const { subagents, turnItems } = yield* projectionStore.getThreadRecords(
1103 threadId,
1104 ["subagents", "turnItems"],
1105 {
1106 omitSettledSubagents: true,
1107 turnItemTypes: ["subagent"],
1108 turnItemStatuses: activeTurnItemStatuses,
1109 },
1110 );
1111 const cutOff = subagents.filter((subagent) => {
1112 if (
1113 subagent.origin !== "provider_native" ||
1114 subagent.driver !== driver ||
1115 !isOrchestrationV2WorkActive(subagent.status) ||
1116 DateTime.isGreaterThan(subagent.startedAt ?? subagent.updatedAt, releasedAt)
1117 ) {
1118 return false;
1119 }
1120 // Record and item are written together; a record without a live item lost its end.
1121 const item = turnItems.find(
1122 (candidate) => candidate.type === "subagent" && candidate.subagentId === subagent.id,
1123 );
1124 return item === undefined || item.providerThreadId === providerThreadId;
1125 });
1126 return { subagents: cutOff, turnItems };
1127 });
1128 let pending = yield* readCutOff;
1129 for (let pass = 0; pass < 3 && pending.subagents.length > 0; pass++) {
1130 const { nodes } = yield* projectionStore.getThreadRecords(threadId, ["nodes"], {
1131 nodeIds: pending.subagents.map((subagent) => subagent.id),
1132 });
1133 const projection = { ...pending, nodes };
1134 for (const subagent of pending.subagents) {
1135 const now = yield* DateTime.now;
1136 const allocateEventId = () =>
1137 idAllocator.allocate.event({ threadId, providerSessionId });
1138 const events = yield* endOrphanedNativeSubagent({
1139 projection,
1140 subagentId: subagent.id,
1141 status: "interrupted",
1142 now,
1143 emitted: [],
1144 allocateEventId,
1145 });
1146 if (subagent.childThreadId !== null) {
1147 const child = yield* projectionStore
1148 .getThreadRecords(subagent.childThreadId, ["nodes", "providerTurns", "turnItems"], {
1149 turnItemRunIds: [null],
1150 turnItemStatuses: activeTurnItemStatuses,
1151 })
1152 .pipe(
1153 Effect.catchTags({
1154 ProjectionStoreThreadNotFoundError: () => Effect.succeed(null),
1155 }),
1156 );
1157 if (child !== null) {
1158 events.push(
1159 ...(yield* endRunlessRootTurns({
1160 threadId: subagent.childThreadId,
1161 providerInstanceId: subagent.providerInstanceId,
1162 projection: child,
1163 status: "interrupted",
1164 now,
1165 emitted: events,
1166 allocateEventId,
1167 })),
1168 );
1169 }
1170 }
1171 // A late update from the dead session's queued events or a resume rejects only this write.
1172 yield* eventSink.writeIfSubagentUnchanged({ expected: subagent, events });
1173 }
1174 pending = yield* readCutOff;
1175 }
1176 if (pending.subagents.length === 0) return;
1177 yield* Effect.logWarning("orchestration-v2.provider-session-release-subagent-conflict", {
1178 providerSessionId,
1179 providerThreadId,
1180 openSubagents: pending.subagents.length,
1181 });
1182 });
1183
1184 const writeReleasedBackgroundWorkEvents = (
1185 released: ReadonlyArray<ReleasedProviderThread>,
1186 releasedAt: DateTime.Utc,
1187 ) =>
1188 Effect.forEach(
1189 released,
1190 (providerThread) =>
1191 writeReleasedRosterEvents(providerThread).pipe(
1192 Effect.andThen(writeReleasedToolCallEvents(providerThread, releasedAt)),
1193 Effect.andThen(writeReleasedSubagentEvents(providerThread, releasedAt)),
1194 ),
1195 { discard: true },
1196 );
1197
1198 const writeReleaseRecords = (input: {
1199 readonly entry: LiveSessionEntry;
1200 readonly reason: ProviderSessionReleaseReason;
1201 readonly detail?: string;
1202 readonly releasedProviderThreads: ReadonlyArray<ReleasedProviderThread>;
1203 readonly releasedAt: DateTime.Utc;
1204 readonly detachedThreadId?: ThreadId;
1205 readonly replaced: boolean;
1206 }) =>
1207 Effect.all(
1208 [
1209 input.replaced
1210 ? Effect.succeed(Exit.void)
1211 : Effect.exit(writeReleasedSessionEvents(input)),
1212 Effect.exit(
1213 writeReleasedRuntimeRequestEvents(input).pipe(
1214 input.entry.requestEventPermit.withPermits(1),
1215 ),
1216 ),
1217 Effect.exit(
1218 writeReleasedBackgroundWorkEvents(input.releasedProviderThreads, input.releasedAt),
1219 ),
1220 ],
1221 { concurrency: 1 },
1222 ).pipe(Effect.flatMap(Exit.asVoidAll));
1223
1224 // Retry after removal from the live map; later releases cannot clear stale ready sessions and approvals.
1225 const retryReleaseRecords = (
1226 input: Omit<Parameters<typeof writeReleaseRecords>[0], "replaced">,
1227 ) => {
1228 const providerSessionId = input.entry.runtime.providerSessionId;
1229 const attempt = Effect.gen(function* () {
1230 const exit = yield* Effect.exit(
1231 sessionOpen.withLock(
1232 providerSessionId,
1233 Effect.gen(function* () {
1234 const replaced = (yield* Ref.get(sessions)).has(sessionKey(providerSessionId));
1235 yield* writeReleaseRecords({ ...input, replaced });
1236 }),
1237 ),
1238 );
1239 if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit;
1240 yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", {
1241 providerSessionId,
1242 cause: exit.cause,
1243 });
1244 // SQL commit failures are defects; retry every cause except interruption.
1245 return yield* Effect.fail(exit.cause);
1246 });
1247 return attempt.pipe(
1248 Effect.retry({
1249 schedule: Schedule.exponential("1 second").pipe(
1250 Schedule.modifyDelay(({ duration }) =>
1251 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1252 ),
1253 ),
1254 }),
1255 Effect.delay("1 second"),
1256 FiberSet.run(releaseRecordRetries),
1257 );
1258 };
1259
1260 const logReleaseFailure =
1261 (providerSessionId: ProviderSessionId) =>
1262 <E, R>(release: Effect.Effect<void, E, R>) =>
1263 release.pipe(
1264 Effect.catchCause((cause) =>
1265 Effect.logWarning("orchestration-v2.provider-session-release-failed", {
1266 providerSessionId,
1267 cause,
1268 }),
1269 ),
1270 );
1271
1272 // A request the event pump is persisting for this runtime lands before
1273 // the cutoff, and once the entry is gone the pump persists no more for
1274 // it.
1275 const removeLiveEntry = Effect.fn("ProviderSessionManager.removeLiveEntry")(
1276 function* (input: {
1277 readonly providerSessionId: ProviderSessionId;
1278 readonly onlyIfIdleGeneration?: number;
1279 readonly keepIfBusyOtherThan?: OwnBusyTurns;
1280 readonly keepIf?: Effect.Effect<boolean>;
1281 }): Effect.fn.Return<readonly [Option.Option<LiveSessionEntry>, DateTime.Utc]> {
1282 const key = sessionKey(input.providerSessionId);
1283 const candidate = (yield* Ref.get(sessions)).get(key);
1284 if (candidate === undefined) {
1285 return [Option.none<LiveSessionEntry>(), yield* DateTime.now] as const;
1286 }
1287 const remove = (keepRequested: boolean) =>
1288 Effect.zip(
1289 Ref.modify(sessions, (current) => {
1290 const existing = current.get(key);
1291 if (existing !== candidate) {
1292 return [existing === undefined ? "gone" : "changed", current] as const;
1293 }
1294 if (
1295 input.onlyIfIdleGeneration !== undefined &&
1296 (existing.busyTurns.size > 0 ||
1297 existing.idleGeneration !== input.onlyIfIdleGeneration)
1298 ) {
1299 return ["kept", current] as const;
1300 }
1301 const own = input.keepIfBusyOtherThan;
1302 if (
1303 keepRequested ||
1304 (own !== undefined &&
1305 [...existing.busyTurns].some((turnKey) => !isOwnBusyTurn(turnKey, own)))
1306 ) {
1307 if (own === undefined) return ["kept", current] as const;
1308 // The caller settled its own turns; one whose end never arrives must not keep
1309 // the session from going idle once the turn it was kept for ends.
1310 const kept = new Map(current);
1311 kept.set(key, {
1312 ...existing,
1313 busyTurns: new Set(
1314 [...existing.busyTurns].filter((turnKey) => !isOwnBusyTurn(turnKey, own)),
1315 ),
1316 });
1317 return ["kept", kept] as const;
1318 }
1319 const updated = new Map(current);
1320 updated.delete(key);
1321 return ["removed", updated] as const;
1322 }),
1323 DateTime.now,
1324 );
1325 const [outcome, releasedAt] = yield* (input.keepIf ?? Effect.succeed(false)).pipe(
1326 Effect.flatMap(remove),
1327 candidate.requestEventPermit.withPermits(1),
1328 );
1329 if (outcome === "changed") return yield* removeLiveEntry(input);
1330 return [
1331 outcome === "removed" ? Option.some(candidate) : Option.none<LiveSessionEntry>(),
1332 releasedAt,
1333 ] as const;
1334 },
1335 );
1336
1337 // Scope close can wedge on a misbehaving adapter finalizer (e.g. a
1338 // provider process that never yields its message stream).
1339 const closeScopeWithin = Effect.fn("ProviderSessionManager.closeScopeWithin")(function* (
1340 scope: Scope.Closeable,
1341 annotations: { readonly providerSessionId?: ProviderSessionId; readonly reason: string },
1342 ) {
1343 const closeFiber = yield* Scope.close(scope, Exit.void).pipe(
1344 Effect.exit,
1345 Effect.forkDetach({ startImmediately: true }),
1346 );
1347 const closeExit = yield* Fiber.join(closeFiber).pipe(
1348 Effect.timeoutOption(RELEASE_SCOPE_CLOSE_TIMEOUT_MS),
1349 );
1350 if (Option.isNone(closeExit)) {
1351 yield* Effect.logWarning("orchestration-v2.provider-session-scope-close-timeout", {
1352 ...annotations,
1353 timeoutMs: RELEASE_SCOPE_CLOSE_TIMEOUT_MS,
1354 });
1355 yield* Fiber.join(closeFiber).pipe(
1356 Effect.flatMap((exit) =>
1357 Exit.isFailure(exit)
1358 ? Effect.logWarning("orchestration-v2.provider-session-scope-close-failed", {
1359 ...annotations,
1360 cause: exit.cause,
1361 })
1362 : Effect.logInfo(
1363 "orchestration-v2.provider-session-scope-close-completed-late",
1364 annotations,
1365 ),
1366 ),
1367 Effect.forkDetach,
1368 );
1369 }
1370 return closeExit;
1371 });
1372
1373 const releaseEntry = (input: {
1374 readonly providerSessionId: ProviderSessionId;
1375 readonly reason: ProviderSessionReleaseReason;
1376 readonly detail?: string;
1377 readonly detachedThreadId?: ThreadId;
1378 readonly cancelIdleFiber?: boolean;
1379 readonly onlyIfIdleGeneration?: number;
1380 readonly keepIfBusyOtherThan?: OwnBusyTurns;
1381 readonly keepIf?: Effect.Effect<boolean>;
1382 readonly gracefulSubscribers?: boolean;
1383 readonly eventStreamCause?: Cause.Cause<ProviderAdapter.ProviderAdapterV2Error>;
1384 }) =>
1385 Effect.acquireUseRelease(
1386 removeLiveEntry(input),
1387 ([entry, releasedAt]) =>
1388 Option.match(entry, {
1389 onNone: () => Effect.void,
1390 onSome: (entry) =>
1391 Effect.gen(function* () {
1392 const detachedThread =
1393 input.detachedThreadId === undefined
1394 ? {}
1395 : { detachedThreadId: input.detachedThreadId };
1396 const releasedProviderThreads = yield* readReleasedProviderThreads({
1397 entry,
1398 reason: input.reason,
1399 ...detachedThread,
1400 });
1401 if (input.cancelIdleFiber !== false) {
1402 yield* cancelIdleFiber(entry.idleFiber);
1403 }
1404 // Queued events can persist tool calls after the release-time scan.
1405 const afterDrain = Effect.forEach(
1406 releasedProviderThreads,
1407 (thread) =>
1408 writeReleasedToolCallEvents(thread, releasedAt).pipe(
1409 Effect.andThen(writeReleasedSubagentEvents(thread, releasedAt)),
1410 ),
1411 { discard: true },
1412 ).pipe(logReleaseFailure(input.providerSessionId));
1413 if (input.gracefulSubscribers === true) {
1414 yield* endSubscribers(entry, afterDrain);
1415 } else if (input.reason === "server_shutdown") {
1416 yield* interruptSubscribers(entry);
1417 } else {
1418 yield* failSubscribers(
1419 entry,
1420 input.detail ?? `Provider session released: ${input.reason}.`,
1421 afterDrain,
1422 input.eventStreamCause,
1423 input.detachedThreadId,
1424 );
1425 }
1426 const closeExit = yield* closeScopeWithin(entry.scope, {
1427 providerSessionId: input.providerSessionId,
1428 reason: input.reason,
1429 });
1430 const records = {
1431 entry,
1432 reason: input.reason,
1433 ...(input.detail === undefined ? {} : { detail: input.detail }),
1434 releasedProviderThreads,
1435 releasedAt,
1436 ...detachedThread,
1437 };
1438 const recorded = yield* Effect.exit(
1439 writeReleaseRecords({ ...records, replaced: false }),
1440 );
1441 if (Exit.isFailure(recorded)) {
1442 yield* retryReleaseRecords(records);
1443 return yield* recorded;
1444 }
1445 if (Option.isSome(closeExit) && Exit.isFailure(closeExit.value)) {
1446 return yield* Effect.failCause(closeExit.value.cause);
1447 }
1448 }).pipe(
1449 withMetrics({
1450 counter: providerSessionsTotal,
1451 attributes: {
1452 provider: entry.runtime.driver,
1453 operation: "release",
1454 reason: input.reason,
1455 },
1456 }),
1457 ),
1458 }),
1459 ([entry]) =>
1460 Option.match(entry, {
1461 onNone: () => Effect.void,
1462 onSome: (entry) =>
1463 // Revoke recorded credentials after process death, except those held by live replacements.
1464 Ref.get(sessions).pipe(
1465 Effect.flatMap((current) =>
1466 Effect.forEach(
1467 entry.mcpCredentialIdByThread,
1468 ([threadId, mcpCredentialId]) => {
1469 // Only holders of this credential may prevent its revocation; stale credentials must not leak it.
1470 const heldElsewhere =
1471 isMcpCredentialReserved(threadId, mcpCredentialId) ||
1472 Array.from(current.values()).some(
1473 (other) =>
1474 other !== entry &&
1475 (other.attachedThreadIds.has(threadId) ||
1476 other.mcpCredentialIdByThread.get(threadId) === mcpCredentialId),
1477 );
1478 return heldElsewhere
1479 ? Effect.void
1480 : clearMcpSession(threadId, mcpCredentialId);
1481 },
1482 { discard: true },
1483 ),
1484 ),
1485 ),
1486 }),
1487 ).pipe(
1488 Effect.catchCause((cause) =>
1489 Effect.fail(
1490 new ProviderSessionReleaseError({
1491 providerSessionId: input.providerSessionId,
1492 reason: input.reason,
1493 cause,
1494 }),
1495 ),
1496 ),
1497 );
1498
1499 const releaseIfStillIdle = Effect.fn("ProviderSessionManager.releaseIfStillIdle")(
1500 function* (input: {
1501 readonly providerSessionId: ProviderSessionId;
1502 readonly generation: number;
1503 }): Effect.fn.Return<void> {
1504 const current = yield* Ref.get(sessions);
1505 const key = sessionKey(input.providerSessionId);
1506 const entry = current.get(key);
1507 if (
1508 entry === undefined ||
1509 entry.busyTurns.size > 0 ||
1510 entry.idleGeneration !== input.generation
1511 ) {
1512 return;
1513 }
1514 // Capture runtime identity before yielding; replacements can reuse the session id.
1515 const probedRuntime = entry.runtime;
1516 const hasPendingWork =
1517 (probedRuntime.hasPendingBackgroundWork === undefined
1518 ? false
1519 : yield* probedRuntime.hasPendingBackgroundWork.pipe(
1520 Effect.catchCause(() => Effect.succeed(false)),
1521 )) ||
1522 // Closing now would skip the archive of a child whose parent has not taken its result.
1523 Option.isSome(
1524 yield* Effect.findFirst(entry.idleThreadUnloads.keys(), (threadId) =>
1525 delegatedChildArchive(threadId).pipe(
1526 Effect.map((child) => child === "awaiting_delivery"),
1527 ),
1528 ),
1529 );
1530 if (hasPendingWork) {
1531 const now = yield* Clock.currentTimeMillis;
1532 const pinnedSinceMs = entry.pinnedSinceMs ?? now;
1533 if (now - pinnedSinceMs < maxIdlePinMs) {
1534 const shouldContinuePin = yield* Ref.modify(sessions, (latest) => {
1535 const latestEntry = latest.get(key);
1536 if (
1537 latestEntry === undefined ||
1538 latestEntry.busyTurns.size > 0 ||
1539 latestEntry.idleGeneration !== input.generation ||
1540 latestEntry.runtime !== probedRuntime
1541 ) {
1542 return [false, latest] as const;
1543 }
1544 const updated = new Map(latest);
1545 updated.set(key, { ...latestEntry, pinnedSinceMs });
1546 return [true, updated] as const;
1547 });
1548 if (!shouldContinuePin) {
1549 return;
1550 }
1551 yield* Effect.logInfo("orchestration-v2.driver-session.idle-release-deferred", {
1552 providerSessionId: input.providerSessionId,
1553 pinnedForMs: now - pinnedSinceMs,
1554 });
1555 // Recheck on this fiber; scheduleIdleReleaseInternal would interrupt it and self-deadlock.
1556 yield* Effect.sleep(Duration.millis(idleTimeoutMs));
1557 return yield* releaseIfStillIdle(input);
1558 }
1559 yield* Effect.logWarning("orchestration-v2.driver-session.idle-release-pin-expired", {
1560 providerSessionId: input.providerSessionId,
1561 pinnedForMs: now - pinnedSinceMs,
1562 });
1563 }
1564 // Closing skips the threads' own idle unloads, so archive delivered children first.
1565 yield* Effect.forEach(
1566 entry.idleThreadUnloads,
1567 ([threadId, pending]) =>
1568 unloadIdleThread({
1569 providerSessionId: input.providerSessionId,
1570 threadId,
1571 generation: pending.generation,
1572 archiveOnly: true,
1573 }),
1574 { discard: true },
1575 );
1576 // Adapter probes yield; guard busyTurns and idleGeneration atomically when removing the entry.
1577 yield* releaseEntry({
1578 providerSessionId: input.providerSessionId,
1579 reason: "idle_timeout",
1580 cancelIdleFiber: false,
1581 onlyIfIdleGeneration: input.generation,
1582 }).pipe(
1583 Effect.catchCause((cause) =>
1584 Effect.logWarning("orchestration-v2.driver-session.idle-release-failed", {
1585 providerSessionId: input.providerSessionId,
1586 cause,
1587 }),
1588 ),
1589 );
1590 },
1591 );
1592
1593 const withActivityError = <A, E, R>(
1594 providerSessionId: ProviderSessionId,
1595 effect: Effect.Effect<A, E, R>,
1596 ): Effect.Effect<A, ProviderSessionActivityError, R> =>
1597 effect.pipe(
1598 Effect.catchCause((cause) =>
1599 Effect.fail(
1600 new ProviderSessionActivityError({
1601 providerSessionId,
1602 cause,
1603 }),
1604 ),
1605 ),
1606 );
1607
1608 // Provider activity restarts the pin budget, so background work that keeps
1609 // reporting progress (a long Claude workflow) is never cut off; only work
1610 // that went silent for maxIdlePinMs is released. Progress events can be
1611 // frequent, so keep this untraced.
1612 const scheduleIdleReleaseInternal = Effect.fnUntraced(function* (
1613 providerSessionId: ProviderSessionId,
1614 options?: { readonly providerActivity?: boolean },
1615 ) {
1616 const key = sessionKey(providerSessionId);
1617 const current = yield* Ref.get(sessions);
1618 const entry = current.get(key);
1619 if (entry === undefined || entry.busyTurns.size > 0) {
1620 return;
1621 }
1622
1623 yield* cancelIdleFiber(entry.idleFiber);
1624 const generation = entry.idleGeneration + 1;
1625 const idleFiber = yield* Effect.sleep(Duration.millis(idleTimeoutMs)).pipe(
1626 Effect.andThen(releaseIfStillIdle({ providerSessionId, generation })),
1627 Effect.forkIn(layerScope),
1628 );
1629 yield* Ref.update(sessions, (latest) => {
1630 const latestEntry = latest.get(key);
1631 if (latestEntry === undefined || latestEntry.busyTurns.size > 0) {
1632 return latest;
1633 }
1634 const updated = new Map(latest);
1635 updated.set(key, {
1636 ...latestEntry,
1637 idleGeneration: generation,
1638 idleFiber,
1639 ...(options?.providerActivity === true ? { pinnedSinceMs: null } : {}),
1640 });
1641 return updated;
1642 });
1643 });
1644
1645 const scheduleIdleRelease = (providerSessionId: ProviderSessionId) =>
1646 withActivityError(providerSessionId, scheduleIdleReleaseInternal(providerSessionId));
1647
1648 const touchActivity = (
1649 providerSessionId: ProviderSessionId,
1650 options?: { readonly providerActivity?: boolean },
1651 ) =>
1652 withActivityError(
1653 providerSessionId,
1654 scheduleIdleReleaseInternal(providerSessionId, options),
1655 );
1656
1657 const attachThread = (input: {
1658 readonly providerSessionId: ProviderSessionId;
1659 readonly threadId: ThreadId;
1660 }) =>
1661 withActivityError(
1662 input.providerSessionId,
1663 Ref.modify(sessions, (current) => {
1664 const entry = current.get(sessionKey(input.providerSessionId));
1665 if (entry === undefined || entry.attachedThreadIds.has(input.threadId)) {
1666 return [undefined, current] as const;
1667 }
1668 const updated = new Map(current);
1669 updated.set(sessionKey(input.providerSessionId), {
1670 ...entry,
1671 attachedThreadIds: new Set([...entry.attachedThreadIds, input.threadId]),
1672 });
1673 return [entry.runtime, updated] as const;
1674 }),
1675 );
1676
1677 const removeThreadAttachment = (input: {
1678 readonly providerSessionId: ProviderSessionId;
1679 readonly threadId: ThreadId;
1680 readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
1681 }) =>
1682 Ref.update(sessions, (current) => {
1683 const key = sessionKey(input.providerSessionId);
1684 const entry = current.get(key);
1685 if (
1686 entry === undefined ||
1687 entry.runtime !== input.runtime ||
1688 !entry.attachedThreadIds.has(input.threadId)
1689 ) {
1690 return current;
1691 }
1692 const attachedThreadIds = new Set(entry.attachedThreadIds);
1693 attachedThreadIds.delete(input.threadId);
1694 const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread);
1695 loadedProviderThreadKeyByThread.delete(input.threadId);
1696 const linkedEnvironmentsByThread = new Map(entry.linkedEnvironmentsByThread);
1697 linkedEnvironmentsByThread.delete(input.threadId);
1698 const updated = new Map(current);
1699 updated.set(key, {
1700 ...entry,
1701 attachedThreadIds,
1702 loadedProviderThreadKeyByThread,
1703 linkedEnvironmentsByThread,
1704 });
1705 return updated;
1706 });
1707
1708 const isProviderThreadLoaded = (input: {
1709 readonly providerSessionId: ProviderSessionId;
1710 readonly threadId: ThreadId;
1711 readonly providerThreadKey: string;
1712 }) =>
1713 Ref.get(sessions).pipe(
1714 Effect.map(
1715 (current) =>
1716 current
1717 .get(sessionKey(input.providerSessionId))
1718 ?.loadedProviderThreadKeyByThread.get(input.threadId) === input.providerThreadKey,
1719 ),
1720 );
1721
1722 const forgetProviderThreadLoaded = (input: {
1723 readonly providerSessionId: ProviderSessionId;
1724 readonly threadId: ThreadId;
1725 readonly runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime;
1726 }) =>
1727 Ref.update(sessions, (current) => {
1728 const key = sessionKey(input.providerSessionId);
1729 const entry = current.get(key);
1730 if (
1731 entry?.runtime !== input.runtime ||
1732 !entry.loadedProviderThreadKeyByThread.has(input.threadId)
1733 ) {
1734 return current;
1735 }
1736 const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread);
1737 loadedProviderThreadKeyByThread.delete(input.threadId);
1738 return new Map(current).set(key, { ...entry, loadedProviderThreadKeyByThread });
1739 });
1740
1741 /** Pass the setting read before a native load; cache hits keep the earlier record. */
1742 const markProviderThreadLoaded = (input: {
1743 readonly providerSessionId: ProviderSessionId;
1744 readonly threadId: ThreadId;
1745 readonly providerThreadKey: string;
1746 readonly linkedEnvironments: boolean | null | undefined;
1747 }) =>
1748 Ref.update(sessions, (current) => {
1749 const key = sessionKey(input.providerSessionId);
1750 const entry = current.get(key);
1751 if (entry === undefined) {
1752 return current;
1753 }
1754 const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread);
1755 loadedProviderThreadKeyByThread.set(input.threadId, input.providerThreadKey);
1756 const linkedEnvironmentsByThread = new Map(entry.linkedEnvironmentsByThread);
1757 if (
1758 input.linkedEnvironments !== undefined &&
1759 !linkedEnvironmentsByThread.has(input.threadId)
1760 ) {
1761 linkedEnvironmentsByThread.set(input.threadId, input.linkedEnvironments);
1762 }
1763 const updated = new Map(current);
1764 updated.set(key, {
1765 ...entry,
1766 loadedProviderThreadKeyByThread,
1767 linkedEnvironmentsByThread,
1768 });
1769 return updated;
1770 });
1771
1772 const loadNatively = <A, E>(load: Effect.Effect<A, E>) =>
1773 readLinkedEnvironments.pipe(
1774 Effect.flatMap((linkedEnvironments) =>
1775 load.pipe(Effect.map((loaded) => ({ loaded, linkedEnvironments }))),
1776 ),
1777 );
1778
1779 const ensureThreadAttached = (input: {
1780 readonly providerSessionId: ProviderSessionId;
1781 readonly threadId: ThreadId;
1782 readonly providerInstanceId: ProviderInstanceId;
1783 }) =>
1784 Effect.suspend(() => {
1785 let attachedTo: ProviderAdapter.ProviderAdapterV2SessionRuntime | undefined;
1786 let preparedForCleanup: PreparedMcpCredential | undefined;
1787 let reservationDropped = false;
1788 const dropReservation = () => {
1789 if (!reservationDropped && preparedForCleanup?.mcpCredentialId !== undefined) {
1790 reservationDropped = true;
1791 dropMcpCredentialReservation(input.threadId, preparedForCleanup.mcpCredentialId);
1792 }
1793 };
1794 // Lock attach through rollback so concurrent attaches cannot observe an attachment about to be undone.
1795 const attach = Effect.gen(function* () {
1796 const attached = yield* attachThread(input).pipe(
1797 // Record without interruption so cleanup undoes only this call's attachment.
1798 Effect.tap((runtime) => Effect.sync(() => (attachedTo = runtime))),
1799 Effect.uninterruptible,
1800 );
1801 if (attached !== undefined) {
1802 const prepared = yield* prepareMcpSession(input.threadId, input.providerInstanceId);
1803 preparedForCleanup = prepared;
1804 if (prepared.mcpCredentialId !== undefined) {
1805 const mcpCredentialId = prepared.mcpCredentialId;
1806 yield* Ref.update(sessions, (current) => {
1807 const key = sessionKey(input.providerSessionId);
1808 const entry = current.get(key);
1809 if (entry === undefined) return current;
1810 const mcpCredentialIdByThread = new Map(entry.mcpCredentialIdByThread);
1811 mcpCredentialIdByThread.set(input.threadId, mcpCredentialId);
1812 const updated = new Map(current);
1813 updated.set(key, { ...entry, mcpCredentialIdByThread });
1814 return updated;
1815 });
1816 }
1817 const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId));
1818 if (entry !== undefined) {
1819 yield* withActivityError(
1820 input.providerSessionId,
1821 writeProviderSessionEvents({
1822 runtime: entry.runtime,
1823 threadIds: [input.threadId],
1824 type: "provider-session.attached",
1825 payload: entry.runtime.providerSession,
1826 }),
1827 );
1828 }
1829 }
1830 }).pipe(
1831 Effect.onError(() =>
1832 attachedTo === undefined
1833 ? Effect.void
1834 : removeThreadAttachment({ ...input, runtime: attachedTo }).pipe(
1835 Effect.andThen(
1836 Effect.suspend(() => {
1837 dropReservation();
1838 // Other live processes hold reused MCP credentials; revocation would break their clients.
1839 if (preparedForCleanup?.issued !== true) return Effect.void;
1840 const mcpCredentialId = preparedForCleanup.mcpCredentialId;
1841 const attachedRuntime = attachedTo;
1842 return Ref.get(sessions).pipe(
1843 Effect.flatMap((current) =>
1844 (mcpCredentialId !== undefined &&
1845 isMcpCredentialReserved(input.threadId, mcpCredentialId)) ||
1846 Array.from(current.values()).some(
1847 (other) =>
1848 other.runtime !== attachedRuntime &&
1849 (other.attachedThreadIds.has(input.threadId) ||
1850 (mcpCredentialId !== undefined &&
1851 other.mcpCredentialIdByThread.get(input.threadId) ===
1852 mcpCredentialId)),
1853 )
1854 ? Effect.void
1855 : clearMcpSession(input.threadId, mcpCredentialId),
1856 ),
1857 );
1858 }),
1859 ),
1860 ),
1861 ),
1862 );
1863 return threadAttachment
1864 .withLock(threadAttachmentKey(input), attach)
1865 .pipe(Effect.ensuring(Effect.sync(dropReservation)));
1866 });
1867
1868 // Marks only the entry this runtime belongs to: a released or replaced session must not
1869 // gain a busy turn that never ends. Succeeds with whether the turn was marked.
1870 const markBusy = (
1871 providerSessionId: ProviderSessionId,
1872 turnKey: string,
1873 runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime,
1874 ) =>
1875 withActivityError(
1876 providerSessionId,
1877 Effect.gen(function* () {
1878 const key = sessionKey(providerSessionId);
1879 const marked = yield* Ref.modify(sessions, (current) => {
1880 const entry = current.get(key);
1881 if (entry === undefined || entry.runtime !== runtime) {
1882 return [undefined, current] as const;
1883 }
1884 const updated = new Map(current);
1885 updated.set(key, {
1886 ...entry,
1887 busyTurns: new Set(entry.busyTurns).add(turnKey),
1888 idleFiber: null,
1889 pinnedSinceMs: null,
1890 });
1891 return [{ idleFiber: entry.idleFiber }, updated] as const;
1892 });
1893 if (marked === undefined) return false;
1894 yield* cancelIdleFiber(marked.idleFiber);
1895 return true;
1896 }),
1897 );
1898
1899 const markIdle = (
1900 providerSessionId: ProviderSessionId,
1901 providerThreadId: ProviderThreadId,
1902 runOrdinal: number,
1903 ) =>
1904 withActivityError(
1905 providerSessionId,
1906 Effect.gen(function* () {
1907 const key = sessionKey(providerSessionId);
1908 yield* Ref.update(sessions, (current) => {
1909 const entry = current.get(key);
1910 if (entry === undefined) {
1911 return current;
1912 }
1913 const busyTurns = new Set(entry.busyTurns);
1914 busyTurns.delete(busyTurnKey(providerThreadId, runOrdinal));
1915 const updated = new Map(current);
1916 updated.set(key, {
1917 ...entry,
1918 busyTurns,
1919 });
1920 return updated;
1921 });
1922 yield* scheduleIdleReleaseInternal(providerSessionId);
1923 yield* scheduleThreadUnload(providerSessionId, providerThreadId);
1924 }),
1925 );
1926
1927 const hasBusyTurn = (entry: LiveSessionEntry, providerThreadId: ProviderThreadId) => {
1928 const prefix = busyTurnPrefix(providerThreadId);
1929 for (const turnKey of entry.busyTurns) {
1930 if (turnKey.startsWith(prefix)) return true;
1931 }
1932 return false;
1933 };
1934
1935 const updateIdleThreadUnload = (
1936 providerSessionId: ProviderSessionId,
1937 threadId: ThreadId,
1938 update: (current: IdleThreadUnload | undefined) => IdleThreadUnload | undefined,
1939 ) =>
1940 Ref.modify(sessions, (current) => {
1941 const key = sessionKey(providerSessionId);
1942 const entry = current.get(key);
1943 if (entry === undefined) return [undefined, current] as const;
1944 const previous = entry.idleThreadUnloads.get(threadId);
1945 const next = update(previous);
1946 const idleThreadUnloads = new Map(entry.idleThreadUnloads);
1947 if (next === undefined) idleThreadUnloads.delete(threadId);
1948 else idleThreadUnloads.set(threadId, next);
1949 const updated = new Map(current);
1950 updated.set(key, { ...entry, idleThreadUnloads });
1951 return [previous, updated] as const;
1952 });
1953
1954 /**
1955 * Called before the thread is resumed or given a turn, so an unload
1956 * cannot land between a resume that found the thread loaded and the turn
1957 * that relies on it.
1958 */
1959 const holdThreadLoaded = Effect.fn("ProviderSessionManager.holdThreadLoaded")(
1960 function* (input: {
1961 readonly providerSessionId: ProviderSessionId;
1962 readonly threadId: ThreadId;
1963 readonly providerThread: OrchestrationV2ProviderThread;
1964 }) {
1965 const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId));
1966 if (
1967 entry === undefined ||
1968 !entry.supportsMultipleProviderThreads ||
1969 entry.exposedRuntime.unloadThread === undefined ||
1970 input.providerThread.nativeThreadRef === null
1971 ) {
1972 return;
1973 }
1974 const previous = yield* updateIdleThreadUnload(
1975 input.providerSessionId,
1976 input.threadId,
1977 (current) => ({
1978 providerThread: input.providerThread,
1979 generation: (current?.generation ?? 0) + 1,
1980 fiber: null,
1981 }),
1982 );
1983 yield* cancelIdleFiber(previous?.fiber ?? null);
1984 },
1985 );
1986
1987 const scheduleThreadUnload = Effect.fn("ProviderSessionManager.scheduleThreadUnload")(
1988 function* (providerSessionId: ProviderSessionId, providerThreadId: ProviderThreadId) {
1989 const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId));
1990 if (entry === undefined || hasBusyTurn(entry, providerThreadId)) return;
1991 const tracked = Array.from(entry.idleThreadUnloads).find(
1992 ([, pending]) => pending.providerThread.id === providerThreadId,
1993 );
1994 if (tracked === undefined) return;
1995 const [threadId, pending] = tracked;
1996 const generation = pending.generation + 1;
1997 const fiber = yield* Effect.sleep(Duration.millis(idleTimeoutMs)).pipe(
1998 Effect.andThen(unloadIdleThread({ providerSessionId, threadId, generation })),
1999 Effect.forkIn(layerScope),
2000 );
2001 const previous = yield* updateIdleThreadUnload(providerSessionId, threadId, (current) =>
2002 current?.generation === pending.generation
2003 ? { ...current, generation, fiber }
2004 : current,
2005 );
2006 yield* cancelIdleFiber(
2007 previous?.generation === pending.generation ? previous.fiber : fiber,
2008 );
2009 },
2010 );
2011
2012 // Provider clients list a delegated child as an ordinary conversation, so a
2013 // finished one is archived once its parent has the result.
2014 const delegatedChildArchive = Effect.fn("ProviderSessionManager.delegatedChildArchive")(
2015 function* (threadId: ThreadId) {
2016 const { thread, runs } = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
2017 runStatuses: [],
2018 includeLatestRun: true,
2019 });
2020 // Follow-up tasks name their run; the initial task is the node the child forked from.
2021 const taskId =
2022 runs.at(-1)?.delegatedTaskId ??
2023 (thread.forkedFrom?.type === "node" ? thread.forkedFrom.nodeId : undefined);
2024 const parentThreadId = thread.lineage.parentThreadId;
2025 if (parentThreadId === null || taskId === undefined) return undefined;
2026 const task = (yield* projectionStore.getThreadRecords(parentThreadId, ["subagents"], {
2027 subagentIds: [taskId],
2028 })).subagents[0];
2029 if (
2030 task?.origin !== "app_owned" ||
2031 task.childThreadId !== threadId ||
2032 task.status !== "completed"
2033 ) {
2034 return undefined;
2035 }
2036 const delivery = task.completionDelivery?.state;
2037 if (delivery === "pending" || delivery === "claimed") return "awaiting_delivery";
2038 return delivery === "delivered" || delivery === "acknowledged" ? "ready" : undefined;
2039 },
2040 (effect) => effect.pipe(Effect.orElseSucceed(() => undefined)),
2041 );
2042
2043 const unloadIdleThread = Effect.fn("ProviderSessionManager.unloadIdleThread")(
2044 function* (input: {
2045 readonly providerSessionId: ProviderSessionId;
2046 readonly threadId: ThreadId;
2047 readonly generation: number;
2048 /** Session release unloads only the threads it would archive, and never waits. */
2049 readonly archiveOnly?: boolean;
2050 }): Effect.fn.Return<void> {
2051 const outcome = yield* threadAttachment.withLock(
2052 threadAttachmentKey(input),
2053 Effect.gen(function* () {
2054 const key = sessionKey(input.providerSessionId);
2055 const entry = (yield* Ref.get(sessions)).get(key);
2056 const pending = entry?.idleThreadUnloads.get(input.threadId);
2057 const unloadThread = entry?.exposedRuntime.unloadThread;
2058 if (
2059 entry === undefined ||
2060 pending === undefined ||
2061 pending.generation !== input.generation ||
2062 unloadThread === undefined ||
2063 !entry.attachedThreadIds.has(input.threadId) ||
2064 hasBusyTurn(entry, pending.providerThread.id)
2065 ) {
2066 return "skipped" as const;
2067 }
2068 // Unloading kills background terminals; keep threads with pending work loaded.
2069 const hasPendingWork =
2070 entry.runtime.hasPendingBackgroundWorkForThread === undefined
2071 ? false
2072 : yield* entry.runtime
2073 .hasPendingBackgroundWorkForThread(pending.providerThread)
2074 .pipe(Effect.catchCause(() => Effect.succeed(false)));
2075 if (hasPendingWork) return "deferred" as const;
2076 const child = yield* delegatedChildArchive(input.threadId);
2077 // Unloading now would miss the archive once the parent takes the result.
2078 if (child === "awaiting_delivery") return "deferred" as const;
2079 const archive = child === "ready";
2080 if (input.archiveOnly === true && !archive) return "skipped" as const;
2081 const unloading = yield* Ref.modify(sessions, (current) => {
2082 const latest = current.get(key);
2083 if (
2084 latest?.runtime !== entry.runtime ||
2085 latest.idleThreadUnloads.get(input.threadId)?.generation !== input.generation
2086 ) {
2087 return [false, current] as const;
2088 }
2089 const loadedProviderThreadKeyByThread = new Map(
2090 latest.loadedProviderThreadKeyByThread,
2091 );
2092 loadedProviderThreadKeyByThread.delete(input.threadId);
2093 const idleThreadUnloads = new Map(latest.idleThreadUnloads);
2094 idleThreadUnloads.delete(input.threadId);
2095 const updated = new Map(current);
2096 updated.set(key, {
2097 ...latest,
2098 loadedProviderThreadKeyByThread,
2099 idleThreadUnloads,
2100 });
2101 return [true, updated] as const;
2102 });
2103 if (!unloading) return "skipped" as const;
2104 const unload = (archiveThread: boolean) =>
2105 unloadThread({
2106 providerThread: pending.providerThread,
2107 archive: archiveThread,
2108 }).pipe(Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS));
2109 yield* unload(archive).pipe(
2110 // A failed or timed-out archive may have left the thread loaded.
2111 Effect.catch((cause) =>
2112 archive
2113 ? Effect.logWarning("orchestration-v2.driver-session.idle-archive-failed", {
2114 providerSessionId: input.providerSessionId,
2115 threadId: input.threadId,
2116 providerThreadId: pending.providerThread.id,
2117 cause,
2118 }).pipe(Effect.andThen(unload(false)))
2119 : Effect.fail(cause),
2120 ),
2121 Effect.tap(() =>
2122 Ref.update(sessions, (current) => {
2123 const latest = current.get(key);
2124 if (latest?.runtime !== entry.runtime) return current;
2125 const linkedEnvironmentsByThread = new Map(latest.linkedEnvironmentsByThread);
2126 linkedEnvironmentsByThread.delete(input.threadId);
2127 return new Map(current).set(key, { ...latest, linkedEnvironmentsByThread });
2128 }),
2129 ),
2130 Effect.catchCause((cause) =>
2131 Effect.logWarning("orchestration-v2.driver-session.idle-unload-failed", {
2132 providerSessionId: input.providerSessionId,
2133 threadId: input.threadId,
2134 providerThreadId: pending.providerThread.id,
2135 cause,
2136 }),
2137 ),
2138 );
2139 return "unloaded" as const;
2140 }),
2141 );
2142 // Wait outside the lock so the next attach is not blocked.
2143 if (outcome === "deferred" && input.archiveOnly !== true) {
2144 yield* Effect.sleep(Duration.millis(idleTimeoutMs));
2145 return yield* unloadIdleThread(input);
2146 }
2147 },
2148 );
2149
2150 /** A shared runtime reloads only this idle thread so the others keep their processes. */
2151 const unloadForLinkedEnvironments = Effect.fn(
2152 "ProviderSessionManager.unloadForLinkedEnvironments",
2153 )(function* (input: {
2154 readonly providerSessionId: ProviderSessionId;
2155 readonly threadId: ThreadId;
2156 readonly providerThread: OrchestrationV2ProviderThread;
2157 }) {
2158 const key = sessionKey(input.providerSessionId);
2159 const entry = (yield* Ref.get(sessions)).get(key);
2160 const loadedWith = entry?.linkedEnvironmentsByThread.get(input.threadId);
2161 const unloadThread = entry?.runtime.unloadThread;
2162 if (
2163 entry === undefined ||
2164 !entry.supportsMultipleProviderThreads ||
2165 unloadThread === undefined ||
2166 loadedWith === undefined ||
2167 input.providerThread.nativeThreadRef === null
2168 ) {
2169 return;
2170 }
2171 const linkedEnvironments = yield* readLinkedEnvironments;
2172 if (typeof linkedEnvironments !== "boolean" || linkedEnvironments === loadedWith) return;
2173 yield* threadAttachment.withLock(
2174 threadAttachmentKey(input),
2175 Effect.gen(function* () {
2176 if (hasBusyTurn(entry, input.providerThread.id)) return;
2177 // Unloading ends the thread's background work; a failed probe keeps it loaded too.
2178 const hasPendingWork =
2179 entry.runtime.hasPendingBackgroundWorkForThread === undefined
2180 ? false
2181 : yield* entry.runtime
2182 .hasPendingBackgroundWorkForThread(input.providerThread, {
2183 includeSubagents: true,
2184 })
2185 .pipe(Effect.catchCause(() => Effect.succeed(true)));
2186 if (hasPendingWork) return;
2187 // Forget the load first so a timed-out unload still resumes natively next turn.
2188 const unloading = yield* Ref.modify(sessions, (current) => {
2189 const latest = current.get(key);
2190 if (
2191 latest?.runtime !== entry.runtime ||
2192 latest.linkedEnvironmentsByThread.get(input.threadId) !== loadedWith ||
2193 hasBusyTurn(latest, input.providerThread.id)
2194 ) {
2195 return [false, current] as const;
2196 }
2197 const loadedProviderThreadKeyByThread = new Map(
2198 latest.loadedProviderThreadKeyByThread,
2199 );
2200 loadedProviderThreadKeyByThread.delete(input.threadId);
2201 return [
2202 true,
2203 new Map(current).set(key, { ...latest, loadedProviderThreadKeyByThread }),
2204 ] as const;
2205 });
2206 if (!unloading) return;
2207 yield* unloadThread({ providerThread: input.providerThread }).pipe(
2208 Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS),
2209 Effect.tap(() =>
2210 Ref.update(sessions, (current) => {
2211 const latest = current.get(key);
2212 if (latest?.runtime !== entry.runtime) return current;
2213 const linkedEnvironmentsByThread = new Map(latest.linkedEnvironmentsByThread);
2214 linkedEnvironmentsByThread.delete(input.threadId);
2215 return new Map(current).set(key, { ...latest, linkedEnvironmentsByThread });
2216 }),
2217 ),
2218 Effect.catchCause((cause) =>
2219 Effect.logWarning("orchestration-v2.driver-session.links-unload-failed", {
2220 providerSessionId: input.providerSessionId,
2221 threadId: input.threadId,
2222 cause,
2223 }),
2224 ),
2225 );
2226 }),
2227 );
2228 });
2229
2230 /** An exclusive runtime configures MCP at open, so it reopens once nothing would be lost. */
2231 const releaseForLinkedEnvironments = Effect.fn(
2232 "ProviderSessionManager.releaseForLinkedEnvironments",
2233 )(function* (entry: LiveSessionEntry) {
2234 if (
2235 entry.supportsMultipleProviderThreads ||
2236 entry.runtime.canResumeAfterRestart === false ||
2237 entry.openedWithLinkedEnvironments === undefined ||
2238 entry.busyTurns.size > 0
2239 ) {
2240 return;
2241 }
2242 const linkedEnvironments = yield* readLinkedEnvironments;
2243 if (
2244 typeof linkedEnvironments !== "boolean" ||
2245 linkedEnvironments === entry.openedWithLinkedEnvironments
2246 ) {
2247 return;
2248 }
2249 const providerSessionId = entry.runtime.providerSessionId;
2250 // Any activity during these reads bumps the generation and keeps the runtime.
2251 const generation = entry.idleGeneration;
2252 const hasPendingWork =
2253 entry.runtime.hasPendingBackgroundWork === undefined
2254 ? false
2255 : yield* entry.runtime.hasPendingBackgroundWork.pipe(
2256 Effect.catchCause(() => Effect.succeed(true)),
2257 );
2258 if (hasPendingWork) return;
2259 // Checked under the request permit so a runless request mid-persist is not cancelled.
2260 const hasUnfinishedRecords = Effect.forEach(entry.attachedThreadIds, (threadId) =>
2261 projectionStore.getThreadRecords(threadId, ["runtimeRequests", "providerThreads"], {
2262 runtimeRequestStatuses: ["pending"],
2263 }),
2264 ).pipe(
2265 Effect.map((projections) =>
2266 projections.some(
2267 (projection) =>
2268 projection.runtimeRequests.some(
2269 (request) =>
2270 request.responseCapability.type === "live" &&
2271 request.responseCapability.providerSessionId === providerSessionId,
2272 ) ||
2273 projection.providerThreads.some(
2274 (providerThread) =>
2275 providerThread.providerSessionId === providerSessionId &&
2276 (providerThread.pendingBackgroundTasks ?? []).length > 0,
2277 ),
2278 ),
2279 ),
2280 Effect.orElseSucceed(() => true),
2281 );
2282 yield* releaseEntry({
2283 providerSessionId,
2284 reason: "idle_timeout",
2285 detail: "Linked environments changed; reopening the provider session.",
2286 onlyIfIdleGeneration: generation,
2287 keepIf: hasUnfinishedRecords,
2288 }).pipe(logReleaseFailure(providerSessionId));
2289 });
2290
2291 const observeActivity = (
2292 providerSessionId: ProviderSessionId,
2293 activity: Effect.Effect<void, ProviderSessionActivityError>,
2294 ) =>
2295 activity.pipe(
2296 Effect.catchCause((cause) =>
2297 Effect.logWarning("orchestration-v2.driver-session.activity-failed", {
2298 providerSessionId,
2299 cause,
2300 }),
2301 ),
2302 );
2303
2304 const makeEventSubscription = Effect.fn("ProviderSessionManager.makeEventSubscription")(
2305 function* (
2306 subscribers: Ref.Ref<
2307 ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
2308 >,
2309 ): Effect.fn.Return<ProviderAdapter.ProviderAdapterV2EventSubscription> {
2310 const queue = yield* Queue.unbounded<ProviderSessionEventSignal, Cause.Done>();
2311 const subscriberId = yield* Ref.getAndUpdate(nextSubscriberId, (value) => value + 1);
2312 yield* Ref.update(subscribers, (current) => {
2313 const updated = new Map(current);
2314 updated.set(subscriberId, queue);
2315 return updated;
2316 });
2317 const close = Ref.modify(subscribers, (current) => {
2318 if (!current.has(subscriberId)) {
2319 return [false, current] as const;
2320 }
2321 const updated = new Map(current);
2322 updated.delete(subscriberId);
2323 return [true, updated] as const;
2324 }).pipe(
2325 Effect.flatMap((removed) =>
2326 removed
2327 ? Queue.clear(queue).pipe(Effect.andThen(Queue.end(queue)), Effect.asVoid)
2328 : Effect.void,
2329 ),
2330 // Detached so the consumer's run failure is not ordered behind release cleanup.
2331 Effect.andThen(
2332 Effect.suspend(() => {
2333 const cleanup = subscriberReleaseCleanup.get(queue);
2334 subscriberReleaseCleanup.delete(queue);
2335 return cleanup === undefined ? Effect.void : Effect.forkDetach(cleanup);
2336 }),
2337 ),
2338 );
2339 const events = Stream.fromQueue(queue).pipe(
2340 Stream.mapEffect((signal) =>
2341 signal.type === "event"
2342 ? Effect.succeed(signal.event)
2343 : Effect.failCause(signal.cause),
2344 ),
2345 Stream.ensuring(close),
2346 );
2347 return { events, close } satisfies ProviderAdapter.ProviderAdapterV2EventSubscription;
2348 },
2349 );
2350
2351 const decorateRuntime = (
2352 runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime,
2353 eventSubscribers: Ref.Ref<
2354 ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
2355 >,
2356 ): ProviderAdapter.ProviderAdapterV2SessionRuntime => {
2357 const providerSessionId = runtime.providerSessionId;
2358 const subscribeEvents = makeEventSubscription(eventSubscribers);
2359 const turnMetrics = (operation: string, model?: string) =>
2360 withMetrics({
2361 counter: providerTurnsTotal,
2362 ...(operation === "send" ? { timer: providerTurnDuration } : {}),
2363 attributes: {
2364 provider: runtime.driver,
2365 operation,
2366 modelFamily: normalizeModelMetricLabel(model),
2367 },
2368 });
2369 return {
2370 ...runtime,
2371 // Pi replaces its session snapshot as metadata changes.
2372 get providerSession() {
2373 return runtime.providerSession;
2374 },
2375 subscribeEvents,
2376 events: Stream.unwrap(
2377 subscribeEvents.pipe(Effect.map((subscription) => subscription.events)),
2378 ),
2379 ensureThread: (input) =>
2380 observeActivity(
2381 providerSessionId,
2382 ensureThreadAttached({
2383 providerSessionId,
2384 threadId: input.threadId,
2385 providerInstanceId: runtime.instanceId,
2386 }),
2387 ).pipe(
2388 Effect.andThen(loadNatively(runtime.ensureThread(input))),
2389 Effect.tap(({ loaded: providerThread, linkedEnvironments }) =>
2390 markProviderThreadLoaded({
2391 providerSessionId,
2392 threadId: input.threadId,
2393 providerThreadKey: providerThreadLoadKey({
2394 runtime,
2395 providerThread,
2396 modelSelection: input.modelSelection,
2397 runtimePolicy: input.runtimePolicy,
2398 }),
2399 linkedEnvironments,
2400 }),
2401 ),
2402 Effect.map(({ loaded }) => loaded),
2403 ),
2404 resumeThread: (input) => {
2405 const threadId = input.threadId ?? input.providerThread.appThreadId;
2406 if (threadId === null || threadId === undefined) {
2407 return runtime.resumeThread(input);
2408 }
2409 const providerThreadKey = providerThreadLoadKey({
2410 runtime,
2411 providerThread: input.providerThread,
2412 ...(input.modelSelection === undefined
2413 ? {}
2414 : { modelSelection: input.modelSelection }),
2415 ...(input.runtimePolicy === undefined ? {} : { runtimePolicy: input.runtimePolicy }),
2416 });
2417 return observeActivity(
2418 providerSessionId,
2419 ensureThreadAttached({
2420 providerSessionId,
2421 threadId,
2422 providerInstanceId: runtime.instanceId,
2423 }),
2424 ).pipe(
2425 Effect.andThen(
2426 holdThreadLoaded({
2427 providerSessionId,
2428 threadId,
2429 providerThread: input.providerThread,
2430 }),
2431 ),
2432 Effect.andThen(
2433 unloadForLinkedEnvironments({
2434 providerSessionId,
2435 threadId,
2436 providerThread: input.providerThread,
2437 }),
2438 ),
2439 Effect.andThen(
2440 isProviderThreadLoaded({ providerSessionId, threadId, providerThreadKey }),
2441 ),
2442 Effect.flatMap((loaded) =>
2443 loaded
2444 ? Effect.succeed({
2445 loaded: input.providerThread,
2446 linkedEnvironments: undefined,
2447 })
2448 : loadNatively(runtime.resumeThread(input)),
2449 ),
2450 Effect.tap(({ loaded: providerThread, linkedEnvironments }) =>
2451 markProviderThreadLoaded({
2452 providerSessionId,
2453 threadId,
2454 providerThreadKey: providerThreadLoadKey({
2455 runtime,
2456 providerThread,
2457 ...(input.modelSelection === undefined
2458 ? {}
2459 : { modelSelection: input.modelSelection }),
2460 ...(input.runtimePolicy === undefined
2461 ? {}
2462 : { runtimePolicy: input.runtimePolicy }),
2463 }),
2464 linkedEnvironments,
2465 }),
2466 ),
2467 Effect.map(({ loaded }) => loaded),
2468 );
2469 },
2470 forkThread: (input) =>
2471 observeActivity(
2472 providerSessionId,
2473 ensureThreadAttached({
2474 providerSessionId,
2475 threadId: input.targetThreadId,
2476 providerInstanceId: runtime.instanceId,
2477 }),
2478 ).pipe(
2479 Effect.andThen(loadNatively(runtime.forkThread(input))),
2480 Effect.tap(({ loaded: providerThread, linkedEnvironments }) =>
2481 markProviderThreadLoaded({
2482 providerSessionId,
2483 threadId: input.targetThreadId,
2484 providerThreadKey: providerThreadLoadKey({
2485 runtime,
2486 providerThread,
2487 ...(input.modelSelection === undefined
2488 ? {}
2489 : { modelSelection: input.modelSelection }),
2490 ...(input.runtimePolicy === undefined
2491 ? {}
2492 : { runtimePolicy: input.runtimePolicy }),
2493 }),
2494 linkedEnvironments,
2495 }),
2496 ),
2497 Effect.map(({ loaded }) => loaded),
2498 ),
2499 startTurn: (input) =>
2500 observeActivity(
2501 providerSessionId,
2502 ensureThreadAttached({
2503 providerSessionId,
2504 threadId: input.threadId,
2505 providerInstanceId: runtime.instanceId,
2506 }),
2507 ).pipe(
2508 // Failed or stopped starts may omit turn.terminal; clear their turns so sessions can idle.
2509 Effect.andThen(
2510 holdThreadLoaded({
2511 providerSessionId,
2512 threadId: input.threadId,
2513 providerThread: input.providerThread,
2514 }),
2515 ),
2516 Effect.andThen(
2517 Effect.acquireUseRelease(
2518 markBusy(
2519 providerSessionId,
2520 busyTurnKey(input.providerThread.id, input.runOrdinal),
2521 runtime,
2522 ).pipe(
2523 Effect.catchCause((cause) =>
2524 Effect.logWarning("orchestration-v2.driver-session.activity-failed", {
2525 providerSessionId,
2526 cause,
2527 }).pipe(Effect.as(true)),
2528 ),
2529 // A release between attach and markBusy leaves nothing to mark: fail the
2530 // start instead of running it on a runtime being torn down.
2531 Effect.flatMap((marked) =>
2532 marked
2533 ? Effect.void
2534 : Effect.fail(
2535 new ProviderAdapter.ProviderAdapterTurnStartError({
2536 driver: runtime.driver,
2537 threadId: input.threadId,
2538 providerThreadId: input.providerThread.id,
2539 runId: input.runId,
2540 cause: "The provider session was released while the turn started.",
2541 }),
2542 ),
2543 ),
2544 ),
2545 () =>
2546 runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)),
2547 (_, exit) =>
2548 Exit.isFailure(exit)
2549 ? observeActivity(
2550 providerSessionId,
2551 markIdle(providerSessionId, input.providerThread.id, input.runOrdinal),
2552 ).pipe(
2553 // A failed start can leave its native thread gone (Claude rejecting a
2554 // resume), so the next start must ask the adapter to resume it again.
2555 Effect.andThen(
2556 forgetProviderThreadLoaded({
2557 providerSessionId,
2558 threadId: input.threadId,
2559 runtime,
2560 }),
2561 ),
2562 )
2563 : Effect.void,
2564 ),
2565 ),
2566 ),
2567 steerTurn: (input) =>
2568 observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
2569 Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))),
2570 ),
2571 // A provider starved of memory can stop answering, so no caller may wait on it unbounded.
2572 interruptTurn: (input) =>
2573 observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
2574 Effect.andThen(
2575 runtime.interruptTurn(input).pipe(
2576 Effect.timeoutOrElse({
2577 duration: PROVIDER_INTERRUPT_TIMEOUT_MS,
2578 orElse: () =>
2579 Effect.fail(
2580 new ProviderAdapter.ProviderAdapterInterruptTimeoutError({
2581 driver: runtime.driver,
2582 providerThreadId: input.providerThread.id,
2583 providerTurnId: input.providerTurnId,
2584 timeoutMs: PROVIDER_INTERRUPT_TIMEOUT_MS,
2585 }),
2586 ),
2587 }),
2588 turnMetrics("interrupt"),
2589 ),
2590 ),
2591 ),
2592 respondToRuntimeRequest: (input) =>
2593 observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe(
2594 Effect.andThen(
2595 runtime
2596 .respondToRuntimeRequest(input)
2597 .pipe(turnMetrics("runtime-request-response")),
2598 ),
2599 ),
2600 };
2601 };
2602
2603 const persistProviderSessionUpdate = Effect.fn(
2604 "ProviderSessionManager.persistProviderSessionUpdate",
2605 )(
2606 function* (
2607 entry: LiveSessionEntry,
2608 event: Extract<
2609 ProviderAdapter.ProviderAdapterV2Event,
2610 { readonly type: "provider_session.updated" }
2611 >,
2612 ) {
2613 const current = (yield* Ref.get(sessions)).get(
2614 sessionKey(entry.runtime.providerSessionId),
2615 );
2616 if (current?.runtime !== entry.runtime) {
2617 return;
2618 }
2619 yield* writeProviderSessionEvents({
2620 runtime: entry.runtime,
2621 threadIds: current.attachedThreadIds,
2622 type: "provider-session.updated",
2623 payload: event.providerSession,
2624 });
2625 },
2626 (effect, entry, _event) =>
2627 effect.pipe(
2628 Effect.catchCause((cause) =>
2629 Effect.logWarning("orchestration-v2.driver-session.status-persist-failed", {
2630 providerSessionId: entry.runtime.providerSessionId,
2631 cause,
2632 }),
2633 ),
2634 ),
2635 );
2636
2637 const makeRetryStallWatch = (entry: LiveSessionEntry) => {
2638 const runningRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2639 const stalledRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2640 const stallTimers = new Map<ProviderTurnId, Fiber.Fiber<void>>();
2641 // Claude and Pi keep the retry row running while a recovered reply streams, so any
2642 // event on the retry's thread counts as the provider being alive.
2643 const activeSinceCheck = new Set<ProviderTurnId>();
2644 const noteThreadActivity = (event: ProviderAdapter.ProviderAdapterV2Event) => {
2645 if (runningRetries.size === 0) return;
2646 const threadId =
2647 event.type === "turn_item.updated"
2648 ? event.turnItem.threadId
2649 : event.type === "message.updated"
2650 ? event.message.threadId
2651 : event.type === "node.updated"
2652 ? event.node.threadId
2653 : event.type === "plan.updated"
2654 ? event.plan.threadId
2655 : undefined;
2656 if (threadId === undefined) return;
2657 for (const item of runningRetries.values()) {
2658 if (item.threadId === threadId) activeSinceCheck.add(item.providerTurnId);
2659 }
2660 };
2661 // Stops the stalled turn; its interrupted terminal is then reported as the retry's failure.
2662 const stopStalledRetry = Effect.fn("ProviderSessionManager.stopStalledRetry")(
2663 function* (item: ProviderRetryItem) {
2664 if (runningRetries.get(item.providerTurnId) !== item) return;
2665 // The terminal this stop causes must not interrupt the stop itself.
2666 stallTimers.delete(item.providerTurnId);
2667 const current = (yield* Ref.get(sessions)).get(
2668 sessionKey(entry.runtime.providerSessionId),
2669 );
2670 if (current?.runtime !== entry.runtime) return;
2671 const { providerThread } = yield* projectionStore.getProviderControlContext(
2672 item.threadId,
2673 { providerThreadId: item.providerThreadId, providerTurnId: item.providerTurnId },
2674 );
2675 if (providerThread === undefined || runningRetries.get(item.providerTurnId) !== item)
2676 return;
2677 stalledRetries.set(item.providerTurnId, item);
2678 yield* entry.exposedRuntime
2679 .interruptTurn({ providerThread, providerTurnId: item.providerTurnId })
2680 .pipe(
2681 // A slow stop can still end the turn later, so that terminal keeps the retry failure.
2682 Effect.catchTags({
2683 ProviderAdapterInterruptTimeoutError: (cause) =>
2684 Effect.logWarning("orchestration-v2.provider-retry-stall-stop-timed-out", {
2685 providerSessionId: entry.runtime.providerSessionId,
2686 cause,
2687 }),
2688 }),
2689 );
2690 },
2691 (effect, item) =>
2692 effect.pipe(
2693 Effect.catchCause((cause) =>
2694 Effect.sync(() => stalledRetries.delete(item.providerTurnId)).pipe(
2695 Effect.andThen(
2696 Effect.logWarning("orchestration-v2.provider-retry-stall-stop-failed", {
2697 providerSessionId: entry.runtime.providerSessionId,
2698 cause,
2699 }),
2700 ),
2701 ),
2702 ),
2703 ),
2704 );
2705 const stopStallTimer = (providerTurnId: ProviderTurnId) => {
2706 const fiber = stallTimers.get(providerTurnId);
2707 stallTimers.delete(providerTurnId);
2708 activeSinceCheck.delete(providerTurnId);
2709 return fiber === undefined ? Effect.void : Fiber.interrupt(fiber);
2710 };
2711 const trackProviderRetry = (
2712 event: ProviderAdapter.ProviderAdapterV2Event,
2713 ): {
2714 readonly event: ProviderAdapter.ProviderAdapterV2Event;
2715 readonly armed?: ProviderRetryItem;
2716 readonly settled?: ProviderTurnId;
2717 } => {
2718 noteThreadActivity(event);
2719 if (event.type === "turn_item.updated") {
2720 const item = event.turnItem;
2721 if (
2722 item.type !== "error" ||
2723 item.retry === undefined ||
2724 item.providerTurnId === null ||
2725 item.providerThreadId === null
2726 )
2727 return { event };
2728 const retryItem = {
2729 ...item,
2730 providerTurnId: item.providerTurnId,
2731 providerThreadId: item.providerThreadId,
2732 retry: item.retry,
2733 };
2734 if (item.status === "running") {
2735 runningRetries.set(retryItem.providerTurnId, retryItem);
2736 return { event, armed: retryItem };
2737 }
2738 // A stall marker is set only once the stop is sent, so only the terminal clears it.
2739 runningRetries.delete(retryItem.providerTurnId);
2740 return { event, settled: retryItem.providerTurnId };
2741 }
2742 if (event.type === "provider_turn.updated") {
2743 if (event.providerTurn.status === "running") return { event };
2744 runningRetries.delete(event.providerTurn.id);
2745 const settled = event.providerTurn.id;
2746 return stalledRetries.has(settled) &&
2747 (event.providerTurn.status === "interrupted" ||
2748 event.providerTurn.status === "cancelled")
2749 ? {
2750 event: { ...event, providerTurn: { ...event.providerTurn, status: "failed" } },
2751 settled,
2752 }
2753 : { event, settled };
2754 }
2755 if (event.type !== "turn.terminal") return { event };
2756 runningRetries.delete(event.providerTurnId);
2757 const stalled = stalledRetries.get(event.providerTurnId);
2758 stalledRetries.delete(event.providerTurnId);
2759 const settled = event.providerTurnId;
2760 if (
2761 stalled === undefined ||
2762 (event.status !== "interrupted" && event.status !== "cancelled")
2763 )
2764 return { event, settled };
2765 return {
2766 settled,
2767 event: {
2768 type: "turn.terminal",
2769 driver: event.driver,
2770 providerThreadId: event.providerThreadId,
2771 providerTurnId: event.providerTurnId,
2772 runOrdinal: event.runOrdinal,
2773 failureItemOrdinal: stalled.ordinal,
2774 status: "failed",
2775 failure: { ...stalled.failure, retryable: false },
2776 retry: stalled.retry,
2777 ...(stalled.startedAt === null ? {} : { retryStartedAt: stalled.startedAt }),
2778 threadDisposition: event.threadDisposition,
2779 ...(event.retainedRuntimeRequestIds === undefined
2780 ? {}
2781 : { retainedRuntimeRequestIds: event.retainedRuntimeRequestIds }),
2782 },
2783 };
2784 };
2785 return {
2786 track: trackProviderRetry,
2787 // Runs for every provider event; spans would be per-event noise. Timers are children
2788 // of the pump fiber, so a released session takes them down.
2789 update: Effect.fnUntraced(function* (tracked: ReturnType<typeof trackProviderRetry>) {
2790 const item = tracked.armed;
2791 if (item === undefined) {
2792 if (tracked.settled !== undefined) yield* stopStallTimer(tracked.settled);
2793 return;
2794 }
2795 yield* stopStallTimer(item.providerTurnId);
2796 const window = Duration.sum(
2797 PROVIDER_RETRY_STALL_TIMEOUT,
2798 Duration.millis(item.retry.retryDelayMs ?? 0),
2799 );
2800 const timer = yield* Effect.gen(function* () {
2801 // Only a thread that stayed silent for a whole window has stalled.
2802 do {
2803 activeSinceCheck.delete(item.providerTurnId);
2804 yield* Effect.sleep(window);
2805 } while (activeSinceCheck.has(item.providerTurnId));
2806 yield* stopStalledRetry(item);
2807 }).pipe(Effect.forkChild({ startImmediately: true }));
2808 stallTimers.set(item.providerTurnId, timer);
2809 }),
2810 };
2811 };
2812
2813 const startEventPump = (entry: LiveSessionEntry, sessionClosed: Deferred.Deferred<void>) => {
2814 let stoppedByProvider = false;
2815 let lastSessionStatus = entry.runtime.providerSession.status;
2816 const retryStalls = makeRetryStallWatch(entry);
2817 return entry.runtime.events.pipe(
2818 // Several adapters never end their event stream, and a parked pump retains the whole session.
2819 Stream.interruptWhen(Deferred.await(sessionClosed)),
2820 Stream.runForEach((nativeEvent) => {
2821 if (shutdownSignal.received) return Effect.void;
2822 // Live output bypasses persistence and subscribers so floods cannot delay domain events.
2823 if (nativeEvent.type === "command_output.delta") {
2824 return commandOutput.append(nativeEvent);
2825 }
2826 if (nativeEvent.type === "prompt_suggestion.updated") {
2827 return SubscriptionRef.update(promptSuggestionStore, (current) => {
2828 const next = new Map(current);
2829 next.delete(nativeEvent.threadId);
2830 next.set(nativeEvent.threadId, {
2831 runId: nativeEvent.runId,
2832 text: nativeEvent.text,
2833 });
2834 for (const threadId of next.keys()) {
2835 if (next.size <= MAX_PROMPT_SUGGESTIONS) break;
2836 next.delete(threadId);
2837 }
2838 return next;
2839 });
2840 }
2841 const tracked = retryStalls.track(nativeEvent);
2842 const event = tracked.event;
2843 const armRetryStall = retryStalls.update(tracked);
2844 const releaseLiveOutput =
2845 event.type === "turn_item.updated" &&
2846 event.turnItem.type === "command_execution" &&
2847 !isOrchestrationV2WorkActive(event.turnItem.status)
2848 ? commandOutput.finish({
2849 threadId: event.turnItem.threadId,
2850 itemId: event.turnItem.id,
2851 })
2852 : Effect.void;
2853 // Metadata can change while idle without extending the runtime's lifetime.
2854 const metadataOnly =
2855 event.type === "provider_session.updated" &&
2856 event.providerSession.status === lastSessionStatus;
2857 if (event.type === "provider_session.updated") {
2858 lastSessionStatus = event.providerSession.status;
2859 if (lastSessionStatus === "stopped") stoppedByProvider = true;
2860 }
2861 return observeActivity(
2862 entry.runtime.providerSessionId,
2863 event.type === "turn.terminal"
2864 ? markIdle(
2865 entry.runtime.providerSessionId,
2866 event.providerThreadId,
2867 event.runOrdinal,
2868 )
2869 : metadataOnly
2870 ? Effect.void
2871 : touchActivity(entry.runtime.providerSessionId, { providerActivity: true }),
2872 ).pipe(
2873 Effect.andThen(armRetryStall),
2874 Effect.andThen(
2875 event.type === "provider_session.updated"
2876 ? persistProviderSessionUpdate(entry, event)
2877 : Effect.void,
2878 ),
2879 Effect.andThen(
2880 Effect.gen(function* () {
2881 if (
2882 event.type === "provider_thread.updated" &&
2883 event.providerThread.status === "error"
2884 ) {
2885 const threadId = event.providerThread.appThreadId;
2886 yield* Ref.update(sessions, (sessions) => {
2887 const key = sessionKey(entry.runtime.providerSessionId);
2888 const latest = sessions.get(key);
2889 if (latest?.runtime !== entry.runtime || threadId === null) return sessions;
2890 const loadedProviderThreadKeyByThread = new Map(
2891 latest.loadedProviderThreadKeyByThread,
2892 );
2893 loadedProviderThreadKeyByThread.delete(threadId);
2894 return new Map(sessions).set(key, {
2895 ...latest,
2896 loadedProviderThreadKeyByThread,
2897 });
2898 });
2899 }
2900 // Requests and recovered background commands can arrive before a run subscribes.
2901 const threadId = sessionScopedArtifactThreadId(event);
2902 if (threadId !== undefined) {
2903 yield* Effect.gen(function* () {
2904 const current = (yield* Ref.get(sessions)).get(
2905 sessionKey(entry.runtime.providerSessionId),
2906 );
2907 if (
2908 current?.runtime !== entry.runtime ||
2909 ((event.type === "provider_thread.updated" ||
2910 (event.type === "turn_item.updated" &&
2911 event.turnItem.type === "command_execution")) &&
2912 !current.attachedThreadIds.has(threadId))
2913 )
2914 return;
2915 yield* providerEventIngestor
2916 .ingestNormalized({
2917 providerSessionId: entry.runtime.providerSessionId,
2918 providerInstanceId: entry.runtime.instanceId,
2919 threadId,
2920 event,
2921 })
2922 .pipe(
2923 Effect.mapError(
2924 (cause) =>
2925 new ProviderAdapter.ProviderAdapterEventStreamError({
2926 driver: entry.runtime.driver,
2927 providerSessionId: entry.runtime.providerSessionId,
2928 raisedByT3: true,
2929 cause,
2930 }),
2931 ),
2932 );
2933 }).pipe(entry.requestEventPermit.withPermits(1));
2934 yield* releaseLiveOutput;
2935 return;
2936 }
2937 yield* publishToSubscribers(entry.eventSubscribers, { type: "event", event });
2938 yield* releaseLiveOutput;
2939 }),
2940 ),
2941 );
2942 }),
2943 Effect.exit,
2944 Effect.flatMap((exit) =>
2945 Effect.gen(function* () {
2946 if (shutdownSignal.received) return;
2947 const current = (yield* Ref.get(sessions)).get(
2948 sessionKey(entry.runtime.providerSessionId),
2949 );
2950 if (current?.runtime !== entry.runtime) {
2951 return;
2952 }
2953 if (stoppedByProvider && Exit.isSuccess(exit)) {
2954 yield* releaseEntry({
2955 providerSessionId: entry.runtime.providerSessionId,
2956 reason: "manual_shutdown",
2957 gracefulSubscribers: true,
2958 }).pipe(logReleaseFailure(entry.runtime.providerSessionId));
2959 return;
2960 }
2961 // Subscribers read an interrupt as server shutdown, so keep it for that.
2962 const cause =
2963 Exit.isFailure(exit) && !Cause.hasInterruptsOnly(exit.cause)
2964 ? exit.cause
2965 : Cause.fail(
2966 new ProviderAdapter.ProviderAdapterEventStreamError({
2967 driver: entry.runtime.driver,
2968 providerSessionId: entry.runtime.providerSessionId,
2969 raisedByT3: true,
2970 cause: "Provider event stream ended unexpectedly.",
2971 }),
2972 );
2973 // lastError is the thread banner, so it gets the same bounded
2974 // message as a failed turn; the raw cause survives only in the log.
2975 yield* Effect.logWarning("orchestration-v2.provider-session-runtime-failed", {
2976 providerSessionId: entry.runtime.providerSessionId,
2977 cause,
2978 });
2979 yield* releaseEntry({
2980 providerSessionId: entry.runtime.providerSessionId,
2981 reason: "runtime_error",
2982 detail: makeProviderFailure({ cause }).message,
2983 eventStreamCause: cause,
2984 }).pipe(logReleaseFailure(entry.runtime.providerSessionId));
2985 }),
2986 ),
2987 Effect.forkIn(layerScope),
2988 );
2989 };
2990
2991 // Shutdown must fit the launcher's 5 s deadline with up to 15 sessions.
2992 const sessionScopes = yield* Scope.make("parallel");
2993 const shutdown = Effect.gen(function* () {
2994 const activeSessions = [...(yield* Ref.get(sessions)).values()];
2995 yield* Effect.forEach(
2996 activeSessions,
2997 (entry) =>
2998 releaseEntry({
2999 providerSessionId: entry.runtime.providerSessionId,
3000 reason: "server_shutdown",
3001 }).pipe(
3002 Effect.catchCause((cause) =>
3003 Effect.logWarning("orchestration-v2.driver-session.shutdown-release-failed", {
3004 providerSessionId: entry.runtime.providerSessionId,
3005 cause,
3006 }),
3007 ),
3008 ),
3009 { concurrency: "unbounded", discard: true },
3010 );
3011 }).pipe(Effect.withSpan("ProviderSessionManager.shutdown"));
3012 yield* Effect.addFinalizer(() =>
3013 shutdown.pipe(
3014 Effect.ensuring(closeScopeWithin(sessionScopes, { reason: "server_shutdown" })),
3015 ),
3016 );
3017
3018 return ProviderSessionManagerV2.of({
3019 shutdown,
3020 open: (input) =>
3021 sessionOpen.withLock(
3022 input.providerSessionId,
3023 Effect.gen(function* () {
3024 const cwd = input.runtimePolicy.cwd;
3025 if (cwd !== null) {
3026 const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe(
3027 Effect.map((stat) => stat.type === "Directory"),
3028 Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")),
3029 );
3030 if (!workspaceIsDirectory) {
3031 return yield* new ProviderWorkspaceMissingError({
3032 threadId: input.threadId,
3033 cwd,
3034 });
3035 }
3036 }
3037 const key = sessionKey(input.providerSessionId);
3038 const live = (yield* Ref.get(sessions)).get(key);
3039 if (live !== undefined) yield* releaseForLinkedEnvironments(live);
3040 const existing = (yield* Ref.get(sessions)).get(key);
3041 if (existing !== undefined) {
3042 const attached = existing.attachedThreadIds.has(input.threadId);
3043 if (!attached && !existing.supportsMultipleProviderThreads) {
3044 return yield* new ProviderSessionOpenError({
3045 instanceId: input.modelSelection.instanceId,
3046 providerSessionId: input.providerSessionId,
3047 cause: `Provider ${existing.runtime.driver} does not support attaching multiple app threads to one session.`,
3048 });
3049 }
3050 if (!attached) yield* agentScope.clear(input.threadId);
3051 yield* ensureThreadAttached({
3052 providerSessionId: input.providerSessionId,
3053 threadId: input.threadId,
3054 providerInstanceId: existing.runtime.instanceId,
3055 });
3056 yield* touchActivity(input.providerSessionId);
3057 return existing.exposedRuntime;
3058 }
3059
3060 const adapter = yield* registry.get(input.modelSelection.instanceId).pipe(
3061 Effect.mapError(
3062 (cause) =>
3063 new ProviderSessionOpenError({
3064 instanceId: input.modelSelection.instanceId,
3065 providerSessionId: input.providerSessionId,
3066 cause,
3067 }),
3068 ),
3069 );
3070 const openedWithLinkedEnvironments = yield* readLinkedEnvironments;
3071 const prepared = yield* prepareMcpSession(
3072 input.threadId,
3073 input.modelSelection.instanceId,
3074 );
3075 const mcpCredentialId = prepared.mcpCredentialId;
3076 let reservationDropped = mcpCredentialId === undefined;
3077 const dropReservation = Effect.sync(() => {
3078 if (!reservationDropped && mcpCredentialId !== undefined) {
3079 reservationDropped = true;
3080 dropMcpCredentialReservation(input.threadId, mcpCredentialId);
3081 }
3082 });
3083 const sessionScope = yield* Scope.fork(sessionScopes);
3084 // Added before the adapter's finalizers so it runs after them: the pump drains shutdown events first.
3085 const sessionClosed = yield* Deferred.make<void>();
3086 yield* Scope.addFinalizer(sessionScope, Deferred.done(sessionClosed, Exit.void));
3087 yield* agentScope.clear(input.threadId);
3088 const runtime = yield* installationGate
3089 .execute(
3090 input.modelSelection.instanceId,
3091 adapter.openSession({
3092 threadId: input.threadId,
3093 providerSessionId: input.providerSessionId,
3094 modelSelection: input.modelSelection,
3095 runtimePolicy: input.runtimePolicy,
3096 promptSuggestions: promptSuggestionsEnabled,
3097 ...(input.resumeFromSession === undefined
3098 ? {}
3099 : { resumeFromSession: input.resumeFromSession }),
3100 ...(input.initialNativeThreadId === undefined
3101 ? {}
3102 : { initialNativeThreadId: input.initialNativeThreadId }),
3103 ...(input.initialProviderItemIdentityVersion === undefined
3104 ? {}
3105 : {
3106 initialProviderItemIdentityVersion:
3107 input.initialProviderItemIdentityVersion,
3108 }),
3109 }),
3110 )
3111 .pipe(
3112 Effect.provideService(Scope.Scope, sessionScope),
3113 // Bound scope close after cleanup; a wedged finalizer must not hold the open lock or interrupter.
3114 Effect.onError(() =>
3115 dropReservation.pipe(
3116 Effect.andThen(
3117 prepared.issued
3118 ? clearMcpSession(input.threadId, mcpCredentialId)
3119 : Effect.void,
3120 ),
3121 Effect.ensuring(
3122 closeScopeWithin(sessionScope, {
3123 providerSessionId: input.providerSessionId,
3124 reason: "open_failed",
3125 }),
3126 ),
3127 ),
3128 ),
3129 Effect.mapError(
3130 (cause) =>
3131 new ProviderSessionOpenError({
3132 instanceId: input.modelSelection.instanceId,
3133 providerSessionId: input.providerSessionId,
3134 cause,
3135 }),
3136 ),
3137 withMetrics({
3138 counter: providerSessionsTotal,
3139 attributes: { provider: adapter.driver, operation: "open" },
3140 }),
3141 );
3142 const eventSubscribers = yield* Ref.make<
3143 ReadonlyMap<number, Queue.Queue<ProviderSessionEventSignal, Cause.Done>>
3144 >(new Map());
3145 const exposedRuntime = decorateRuntime(runtime, eventSubscribers);
3146 const entry: LiveSessionEntry = {
3147 attachedThreadIds: new Set([input.threadId]),
3148 loadedProviderThreadKeyByThread: new Map(),
3149 openedWithLinkedEnvironments,
3150 linkedEnvironmentsByThread: new Map(),
3151 mcpCredentialIdByThread:
3152 mcpCredentialId === undefined
3153 ? new Map()
3154 : new Map([[input.threadId, mcpCredentialId]]),
3155 supportsMultipleProviderThreads:
3156 runtime.providerSession.capabilities.sessions
3157 .supportsMultipleProviderThreadsPerSession,
3158 runtime,
3159 exposedRuntime,
3160 eventSubscribers,
3161 requestEventPermit: yield* Semaphore.make(1),
3162 scope: sessionScope,
3163 idleGeneration: 0,
3164 busyTurns: new Set(),
3165 idleFiber: null,
3166 pinnedSinceMs: null,
3167 idleThreadUnloads: new Map(),
3168 };
3169 yield* Ref.update(sessions, (current) => {
3170 const updated = new Map(current);
3171 updated.set(key, entry);
3172 return updated;
3173 });
3174 yield* dropReservation;
3175 yield* withActivityError(
3176 input.providerSessionId,
3177 writeProviderSessionEvents({
3178 runtime,
3179 threadIds: [input.threadId],
3180 type: "provider-session.attached",
3181 payload: runtime.providerSession,
3182 }),
3183 ).pipe(
3184 // Release on interruption too; no event pump or idle timer exists to clean up yet.
3185 Effect.onError((cause) =>
3186 releaseEntry(
3187 Cause.hasInterruptsOnly(cause)
3188 ? {
3189 providerSessionId: input.providerSessionId,
3190 reason: "manual_shutdown",
3191 detail: "The provider session start was interrupted.",
3192 }
3193 : {
3194 providerSessionId: input.providerSessionId,
3195 reason: "runtime_error",
3196 detail: "Failed to persist the provider-session attachment.",
3197 },
3198 ).pipe(logReleaseFailure(input.providerSessionId)),
3199 ),
3200 );
3201 yield* startEventPump(entry, sessionClosed);
3202 yield* scheduleIdleRelease(input.providerSessionId);
3203 return exposedRuntime;
3204 }),
3205 ),
3206 isLive: (providerSessionId) =>
3207 Ref.get(sessions).pipe(
3208 Effect.map((current) => current.has(sessionKey(providerSessionId))),
3209 ),
3210 get: Effect.fn("ProviderSessionManager.get")(
3211 function* (providerSessionId) {
3212 const entry = (yield* Ref.get(sessions)).get(sessionKey(providerSessionId));
3213 if (entry === undefined) {
3214 return Option.none<ProviderAdapter.ProviderAdapterV2SessionRuntime>();
3215 }
3216 yield* touchActivity(providerSessionId);
3217 return Option.some(entry.exposedRuntime);
3218 },
3219 (effect, providerSessionId) =>
3220 effect.pipe(
3221 Effect.mapError(
3222 (cause) =>
3223 new ProviderSessionLookupError({
3224 providerSessionId,
3225 cause,
3226 }),
3227 ),
3228 ),
3229 ),
3230 close: (providerSessionId) =>
3231 releaseEntry({ providerSessionId, reason: "manual_shutdown" }).pipe(
3232 Effect.mapError(
3233 (cause) =>
3234 new ProviderSessionCloseError({
3235 providerSessionId,
3236 cause,
3237 }),
3238 ),
3239 ),
3240 closeInstance: Effect.fn("ProviderSessionManager.closeInstance")(
3241 function* (instanceId) {
3242 const active = [...(yield* Ref.get(sessions)).values()].filter(
3243 (entry) => entry.runtime.instanceId === instanceId,
3244 );
3245 const outcomes = yield* Effect.forEach(
3246 active,
3247 (entry) =>
3248 releaseEntry({
3249 providerSessionId: entry.runtime.providerSessionId,
3250 reason: "manual_shutdown",
3251 detail: `Provider instance ${instanceId} logged out.`,
3252 }).pipe(Effect.exit),
3253 { concurrency: "unbounded" },
3254 );
3255 const failure = outcomes.find(Exit.isFailure);
3256 if (failure !== undefined && Exit.isFailure(failure)) {
3257 return yield* Effect.failCause(failure.cause);
3258 }
3259 },
3260 (effect, instanceId) =>
3261 effect.pipe(
3262 Effect.mapError(
3263 (cause) =>
3264 new ProviderSessionCloseError({
3265 providerSessionId: ProviderSessionId.make(
3266 `provider-session:provider-instance:${instanceId}`,
3267 ),
3268 cause,
3269 }),
3270 ),
3271 ),
3272 ),
3273 release: releaseEntry,
3274 releaseUnlessBusy: Effect.fn("ProviderSessionManager.releaseUnlessBusy")(function* (input) {
3275 const key = sessionKey(input.providerSessionId);
3276 const before = (yield* Ref.get(sessions)).get(key);
3277 yield* releaseEntry({
3278 providerSessionId: input.providerSessionId,
3279 reason: input.reason,
3280 ...(input.detail === undefined ? {} : { detail: input.detail }),
3281 keepIfBusyOtherThan: {
3282 providerThreadIds: input.ownProviderThreadIds,
3283 throughRunOrdinal: input.throughRunOrdinal,
3284 },
3285 ...(input.keepIf === undefined ? {} : { keepIf: input.keepIf }),
3286 });
3287 // Scope, not presence: open can install a replacement under the same id meanwhile, and
3288 // a kept entry is rewritten with the same scope.
3289 const released =
3290 before !== undefined && (yield* Ref.get(sessions)).get(key)?.scope !== before.scope;
3291 // The stopped turn's markBusy cancelled the idle timer and its markIdle may never come.
3292 if (!released) yield* scheduleIdleReleaseInternal(input.providerSessionId);
3293 return released;
3294 }),
3295 otherWorkingThreads: Effect.fn("ProviderSessionManager.otherWorkingThreads")(
3296 function* (input) {
3297 const entry = (yield* Ref.get(sessions)).get(sessionKey(input.providerSessionId));
3298 if (entry === undefined || !entry.supportsMultipleProviderThreads) return [];
3299 return yield* Effect.filter(
3300 [...entry.attachedThreadIds].filter((threadId) => threadId !== input.threadId),
3301 (threadId) =>
3302 // Runs, not the shell's activeRunId: the shell omits silent runs such as idle
3303 // compaction. Background work outlives its run and still needs the process.
3304 Effect.zipWith(
3305 projectionStore.getThreadRecords(threadId, ["runs"]),
3306 projectionStore.getThreadShell(threadId),
3307 ({ runs }, shell) =>
3308 runs.some((run) => WORKING_RUN_STATUSES.has(run.status)) ||
3309 (shell?.pendingBackgroundTasks?.length ?? 0) > 0,
3310 ).pipe(
3311 // An unreadable thread may still be working; never kill it unseen.
3312 Effect.orElseSucceed(() => true),
3313 ),
3314 );
3315 },
3316 ),
3317 detach: Effect.fn("ProviderSessionManager.detach")(
3318 function* (input) {
3319 const key = sessionKey(input.providerSessionId);
3320 const currentEntry = (yield* Ref.get(sessions)).get(key);
3321 let detachedProviderThreads: ReadonlyArray<OrchestrationV2ProviderThread> = [];
3322 if (currentEntry?.supportsMultipleProviderThreads === true) {
3323 const projection = yield* Effect.option(
3324 projectionStore.getThreadRecords(input.threadId, [
3325 "providerThreads",
3326 "providerTurns",
3327 ]),
3328 );
3329 if (Option.isSome(projection)) {
3330 const providerThreads = new Map(
3331 projection.value.providerThreads
3332 .filter((thread) => thread.providerSessionId === input.providerSessionId)
3333 .map((thread) => [thread.id, thread] as const),
3334 );
3335 detachedProviderThreads = [...providerThreads.values()];
3336 const activeTurns = projection.value.providerTurns.filter(
3337 (turn) => turn.status === "running" && providerThreads.has(turn.providerThreadId),
3338 );
3339 yield* Effect.forEach(
3340 activeTurns,
3341 (turn) =>
3342 currentEntry.exposedRuntime
3343 .interruptTurn({
3344 providerThread: providerThreads.get(turn.providerThreadId)!,
3345 providerTurnId: turn.id,
3346 })
3347 .pipe(
3348 Effect.catchCause((cause) =>
3349 Effect.logWarning(
3350 "orchestration-v2.driver-session.detach-interrupt-failed",
3351 {
3352 providerSessionId: input.providerSessionId,
3353 threadId: input.threadId,
3354 providerTurnId: turn.id,
3355 cause,
3356 },
3357 ),
3358 ),
3359 ),
3360 { concurrency: 1, discard: true },
3361 );
3362 }
3363 }
3364 const detachResult = yield* Ref.modify(sessions, (current) => {
3365 const entry = current.get(key);
3366 if (entry === undefined || !entry.attachedThreadIds.has(input.threadId)) {
3367 return [
3368 Option.none<{
3369 readonly entry: LiveSessionEntry;
3370 readonly idleUnloadFiber: Fiber.Fiber<void, never> | null;
3371 }>(),
3372 current,
3373 ] as const;
3374 }
3375 const attachedThreadIds = new Set(entry.attachedThreadIds);
3376 attachedThreadIds.delete(input.threadId);
3377 const loadedProviderThreadKeyByThread = new Map(
3378 entry.loadedProviderThreadKeyByThread,
3379 );
3380 loadedProviderThreadKeyByThread.delete(input.threadId);
3381 const linkedEnvironmentsByThread = new Map(entry.linkedEnvironmentsByThread);
3382 linkedEnvironmentsByThread.delete(input.threadId);
3383 const mcpCredentialIdByThread =
3384 input.revokeMcpCredential === true
3385 ? (() => {
3386 const pruned = new Map(entry.mcpCredentialIdByThread);
3387 pruned.delete(input.threadId);
3388 return pruned;
3389 })()
3390 : entry.mcpCredentialIdByThread;
3391 const idleThreadUnloads = new Map(entry.idleThreadUnloads);
3392 idleThreadUnloads.delete(input.threadId);
3393 const updatedEntry = {
3394 ...entry,
3395 attachedThreadIds,
3396 loadedProviderThreadKeyByThread,
3397 linkedEnvironmentsByThread,
3398 mcpCredentialIdByThread,
3399 idleThreadUnloads,
3400 };
3401 const updated = new Map(current);
3402 updated.set(key, updatedEntry);
3403 return [
3404 Option.some({
3405 entry: updatedEntry,
3406 idleUnloadFiber: entry.idleThreadUnloads.get(input.threadId)?.fiber ?? null,
3407 }),
3408 updated,
3409 ] as const;
3410 });
3411 if (Option.isSome(detachResult)) {
3412 yield* cancelIdleFiber(detachResult.value.idleUnloadFiber);
3413 }
3414 const detached = Option.map(detachResult, (result) => result.entry);
3415 // Plain detaches preserve credentials held by live shared providers across workspace handoffs.
3416 if (input.revokeMcpCredential === true) {
3417 yield* clearMcpSession(input.threadId);
3418 }
3419 if (Option.isNone(detached)) {
3420 return;
3421 }
3422 if (
3423 detached.value.attachedThreadIds.size === 0 &&
3424 !detached.value.supportsMultipleProviderThreads
3425 ) {
3426 yield* releaseEntry({
3427 providerSessionId: input.providerSessionId,
3428 reason: "manual_shutdown",
3429 ...(input.detail === undefined ? {} : { detail: input.detail }),
3430 detachedThreadId: input.threadId,
3431 });
3432 return;
3433 }
3434 const unloadThread = detached.value.exposedRuntime.unloadThread;
3435 if (detached.value.supportsMultipleProviderThreads && unloadThread !== undefined) {
3436 // The projection includes native children owned by other app threads;
3437 // lock each owner so an attached child stays loaded and later attaches wait.
3438 for (const providerThread of detachedProviderThreads) {
3439 if (providerThread.nativeThreadRef === null) continue;
3440 const ownerThreadId = providerThread.appThreadId ?? input.threadId;
3441 yield* threadAttachment.withLock(
3442 threadAttachmentKey({
3443 providerSessionId: input.providerSessionId,
3444 threadId: ownerThreadId,
3445 }),
3446 Effect.gen(function* () {
3447 const entry = (yield* Ref.get(sessions)).get(key);
3448 if (
3449 entry?.runtime !== detached.value.runtime ||
3450 entry.attachedThreadIds.has(ownerThreadId)
3451 ) {
3452 return;
3453 }
3454 yield* unloadThread({ providerThread }).pipe(
3455 Effect.timeout(UNLOAD_THREAD_TIMEOUT_MS),
3456 Effect.catchCause((cause) =>
3457 Effect.logWarning("orchestration-v2.driver-session.detach-unload-failed", {
3458 providerSessionId: input.providerSessionId,
3459 threadId: ownerThreadId,
3460 providerThreadId: providerThread.id,
3461 cause,
3462 }),
3463 ),
3464 );
3465 }),
3466 );
3467 }
3468 }
3469 yield* scheduleIdleRelease(input.providerSessionId);
3470 },
3471 (effect, input) =>
3472 effect.pipe(
3473 Effect.catchCause((cause) =>
3474 Effect.fail(
3475 new ProviderSessionReleaseError({
3476 providerSessionId: input.providerSessionId,
3477 reason: "manual_shutdown",
3478 cause,
3479 }),
3480 ),
3481 ),
3482 ),
3483 ),
3484 forgetLoadedProviderThread: (input) =>
3485 Ref.update(sessions, (current) => {
3486 const key = sessionKey(input.providerSessionId);
3487 const entry = current.get(key);
3488 // Only the matching load: a different key belongs to a later load of this thread.
3489 if (
3490 entry === undefined ||
3491 entry.loadedProviderThreadKeyByThread.get(input.threadId) !==
3492 providerThreadLoadKey({ ...input, runtime: entry.runtime })
3493 ) {
3494 return current;
3495 }
3496 const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread);
3497 loadedProviderThreadKeyByThread.delete(input.threadId);
3498 return new Map(current).set(key, { ...entry, loadedProviderThreadKeyByThread });
3499 }),
3500 // Stored suggestions can outlive a deleted thread, so check the thread before serving one.
3501 subscribePromptSuggestion: (threadId) =>
3502 SubscriptionRef.changes(promptSuggestionStore).pipe(
3503 Stream.map((suggestions) => suggestions.get(threadId) ?? null),
3504 Stream.changes,
3505 Stream.mapEffect((suggestion) =>
3506 suggestion === null
3507 ? Effect.succeed(null)
3508 : projectionStore.getThread(threadId).pipe(
3509 Effect.map((thread) => (thread.deletedAt === null ? suggestion : null)),
3510 Effect.orElseSucceed(() => null),
3511 ),
3512 ),
3513 Stream.changes,
3514 ),
3515 } satisfies ProviderSessionManagerV2Shape);
3516 }),
3517 );
3518
3519 export const layer = layerWithOptions();