The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,266 lines 48.9 KB
Raw
1 import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
2 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3 import {
4 CommandId,
5 isProviderNativeSubagentThread,
6 type OrchestrationV2DomainEvent,
7 type ProviderThreadId,
8 type RunAttemptId,
9 type OrchestrationV2RestartCancelledBackgroundWork,
10 type OrchestrationV2Subagent,
11 type OrchestrationV2ThreadProjection,
12 isOrchestrationV2WorkActive,
13 ThreadId,
14 } from "@t3tools/contracts";
15 import * as Cause from "effect/Cause";
16 import * as Context from "effect/Context";
17 import * as DateTime from "effect/DateTime";
18 import * as Effect from "effect/Effect";
19 import * as Layer from "effect/Layer";
20 import * as Option from "effect/Option";
21 import * as Schema from "effect/Schema";
22
23 import * as EffectOutbox from "./EffectOutbox.ts";
24 import * as EventSink from "./EventSink.ts";
25 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
26 import * as ProjectionStore from "./ProjectionStore.ts";
27 import * as ServerSettings from "../serverSettings.ts";
28 import {
29 restartContinuationBlocked,
30 restartContinuationRun,
31 restartStopRequested,
32 } from "./RestartContinuation.ts";
33 import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
34 import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
35 import {
36 endOrphanedNativeSubagent,
37 endRunlessRootTurns,
38 } from "@t3tools/provider-core/server/subagentProjection";
39 import {
40 awaitingProviderThreadId,
41 cancelledRosterTaskWork,
42 cancelledTurnItemWork,
43 mergeRestartCancelledBackgroundWork,
44 neverStarted,
45 } from "./RestartBackgroundNote.ts";
46
47 export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRuntimeRecoveryError>()(
48 "ProviderRuntimeRecoveryError",
49 {
50 operation: Schema.Literals(["read-projections", "reconcile", "drain-outbox"]),
51 threadId: Schema.optional(ThreadId),
52 cause: Schema.Defect(),
53 },
54 ) {
55 override get message(): string {
56 return `Provider runtime recovery failed during ${this.operation}.`;
57 }
58 }
59
60 const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);
61
62 const MAX_RESENDS = 3;
63
64 export interface ProviderRuntimeRecoverySummary {
65 readonly terminalizedRuns: number;
66 readonly stoppedSessions: number;
67 readonly closedRequests: number;
68 readonly retiredEffects: number;
69 readonly requeuedEffects: number;
70 }
71
72 export interface ProviderRuntimeReconciliationSummary {
73 readonly terminalizedRuns: number;
74 readonly stoppedSessions: number;
75 readonly closedRequests: number;
76 readonly retiredEffects: number;
77 readonly requeuedEffects: number;
78 }
79
80 export class ProviderRuntimeRecoveryService extends Context.Service<
81 ProviderRuntimeRecoveryService,
82 {
83 readonly reconcile: (
84 trigger: "startup" | "shutdown",
85 ) => Effect.Effect<ProviderRuntimeReconciliationSummary, ProviderRuntimeRecoveryError>;
86 readonly prepareForShutdown: Effect.Effect<void, ProviderRuntimeRecoveryError>;
87 readonly recover: Effect.Effect<ProviderRuntimeRecoverySummary, ProviderRuntimeRecoveryError>;
88 }
89 >()("t3/orchestration-v2/ProviderRuntimeRecoveryService") {}
90
91 function nonterminalRuns(projection: ProjectionStore.ProjectionRuntimeRecoveryState) {
92 return projection.runs.filter((run) => {
93 const status: string = run.status;
94 return (
95 status === "preparing" ||
96 run.status === "starting" ||
97 run.status === "running" ||
98 run.status === "waiting"
99 );
100 });
101 }
102
103 function isBackgroundCapableTurnItemType(type: string): boolean {
104 return type === "command_execution" || type === "dynamic_tool" || type === "subagent";
105 }
106
107 function isNonterminalTurnItemStatus(status: string): boolean {
108 return status === "pending" || status === "running" || status === "waiting";
109 }
110
111 /**
112 * A delegate_task child. Its own thread is reconciled and continued on its own
113 * (here, or in the linked environment it runs in) and reports back through
114 * the app, so it is not provider background work.
115 */
116 function isAppOwnedDelegation(task: {
117 readonly origin: OrchestrationV2Subagent["origin"];
118 readonly childThreadId: ThreadId | null;
119 readonly remoteChild?: OrchestrationV2Subagent["remoteChild"];
120 }): boolean {
121 return (
122 task.origin === "app_owned" && (task.childThreadId !== null || task.remoteChild !== undefined)
123 );
124 }
125
126 function isAppOwnedDelegationItem(
127 item: OrchestrationV2ThreadProjection["turnItems"][number],
128 ): boolean {
129 return item.type === "subagent" && isAppOwnedDelegation(item);
130 }
131
132 function providerThreadHasPendingBackgroundTasks(
133 providerThread: OrchestrationV2ThreadProjection["providerThreads"][number],
134 ): boolean {
135 return (providerThread.pendingBackgroundTasks?.length ?? 0) > 0;
136 }
137
138 function resolveStaleBackgroundItemProviderInstanceId(
139 item: OrchestrationV2ThreadProjection["turnItems"][number],
140 projection: ProjectionStore.ProjectionRuntimeRecoveryState,
141 ): OrchestrationV2ThreadProjection["thread"]["providerInstanceId"] {
142 if (item.runId !== null) {
143 const run = projection.runs.find((candidate) => candidate.id === item.runId);
144 if (run !== undefined) {
145 return run.providerInstanceId;
146 }
147 }
148 if (item.type === "subagent") {
149 return item.providerInstanceId;
150 }
151 if (item.providerThreadId !== null && item.providerThreadId !== undefined) {
152 const providerThread = projection.providerThreads.find(
153 (candidate) => candidate.id === item.providerThreadId,
154 );
155 if (providerThread !== undefined) {
156 return providerThread.providerInstanceId;
157 }
158 }
159 return projection.providerThreads[0]?.providerInstanceId ?? projection.thread.providerInstanceId;
160 }
161
162 const cancelledItem = (
163 item: OrchestrationV2ThreadProjection["turnItems"][number],
164 now: DateTime.Utc,
165 ): OrchestrationV2ThreadProjection["turnItems"][number] =>
166 item.type === "secret_request" && item.secretStatus === "pending"
167 ? { ...item, status: "cancelled", secretStatus: "cancelled", completedAt: now, updatedAt: now }
168 : {
169 ...item,
170 ...("streaming" in item ? { streaming: false } : {}),
171 status: "cancelled",
172 completedAt: now,
173 updatedAt: now,
174 };
175
176 function latestStartedRun(
177 projection: ProjectionStore.ProjectionRuntimeRecoveryState,
178 providerThreadId: ProviderThreadId,
179 ) {
180 return projection.runs.reduce<OrchestrationV2ThreadProjection["runs"][number] | undefined>(
181 (latest, run) =>
182 run.providerThreadId === providerThreadId &&
183 run.status !== "queued" &&
184 run.status !== "rolled_back" &&
185 !neverStarted(run) &&
186 (latest === undefined || runRanAfter(run, latest))
187 ? run
188 : latest,
189 undefined,
190 );
191 }
192
193 export const make = Effect.gen(function* () {
194 const settings = yield* ServerSettings.ServerSettingsService;
195 const projections = yield* ProjectionStore.ProjectionStoreV2;
196 const eventSink = yield* EventSink.EventSinkV2;
197 const ids = yield* IdAllocator.IdAllocatorV2;
198 const outbox = yield* EffectOutbox.EffectOutboxV2;
199 type RecoverySession = ProjectionStore.ProjectionRuntimeRecoveryState["providerSessions"][number];
200 /** Shared sessions stop on their first recovered thread; later threads still decide wakes from pre-recovery liveness. */
201 const sessionsLiveAtStart = (
202 projection: ProjectionStore.ProjectionRuntimeRecoveryState,
203 stoppedThisPass: ReadonlyMap<string, RecoverySession>,
204 ) => {
205 const sessions = new Map(projection.providerSessions.map((session) => [session.id, session]));
206 for (const providerThread of projection.providerThreads) {
207 const stopped =
208 providerThread.providerSessionId === null
209 ? undefined
210 : stoppedThisPass.get(providerThread.providerSessionId);
211 if (stopped !== undefined) sessions.set(stopped.id, stopped);
212 }
213 return [...sessions.values()];
214 };
215
216 /**
217 * A run is marked running before its provider accepts the prompt, so only a provider turn of
218 * one of its attempts proves the prompt arrived. Recovery treats the others as never started.
219 */
220 const rewindUnacceptedRuns = Effect.fn("ProviderRuntimeRecoveryService.rewindUnacceptedRuns")(
221 function* (projection: ProjectionStore.ProjectionRuntimeRecoveryState) {
222 const runIdByAttempt = new Map(
223 projection.runs.flatMap((run) =>
224 (run.status === "running" || (run.status === "starting" && run.startedAt !== null)) &&
225 run.activeAttemptId !== null
226 ? [[run.activeAttemptId, run.id] as const]
227 : [],
228 ),
229 );
230 if (runIdByAttempt.size === 0) return projection;
231 const unaccepted = new Set(runIdByAttempt.values());
232 const accept = (turns: ReadonlyArray<{ readonly runAttemptId: RunAttemptId | null }>) => {
233 for (const turn of turns) {
234 const runId =
235 turn.runAttemptId === null ? undefined : runIdByAttempt.get(turn.runAttemptId);
236 if (runId !== undefined) unaccepted.delete(runId);
237 }
238 };
239 accept(projection.providerTurns);
240 if (unaccepted.size === 0) return projection;
241 for (const attempt of projection.attempts) {
242 if (unaccepted.has(attempt.runId)) runIdByAttempt.set(attempt.id, attempt.runId);
243 }
244 // The recovery projection omits ended provider turns, such as one a steer replaced.
245 accept(
246 (yield* projections
247 .getThreadRecords(projection.thread.id, ["providerTurns"], {
248 providerTurnAttemptIds: [...runIdByAttempt].flatMap(([attemptId, runId]) =>
249 unaccepted.has(runId) ? [attemptId] : [],
250 ),
251 })
252 .pipe(
253 Effect.mapError(
254 (cause) =>
255 new ProviderRuntimeRecoveryError({
256 operation: "read-projections",
257 threadId: projection.thread.id,
258 cause,
259 }),
260 ),
261 )).providerTurns,
262 );
263 return {
264 ...projection,
265 runs: projection.runs.map((run) =>
266 unaccepted.has(run.id) ? { ...run, status: "starting" as const, startedAt: null } : run,
267 ),
268 };
269 },
270 );
271
272 const reconcileProjection = Effect.fn("ProviderRuntimeRecoveryService.reconcileProjection")(
273 function* (
274 stored: ProjectionStore.ProjectionRuntimeRecoveryState,
275 trigger: "startup" | "shutdown",
276 continueAfterRestart: boolean,
277 stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
278 ) {
279 const now = yield* DateTime.now;
280 const projection = yield* rewindUnacceptedRuns(stored);
281 let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
282 for (const run of nonterminalRuns(projection)) {
283 if (run.status === "waiting") {
284 const checkpointEffects = yield* outbox
285 .listByCommandId(CommandId.make(`command:effect:checkpoint.capture:${run.id}`))
286 .pipe(
287 Effect.mapError(
288 (cause) =>
289 new ProviderRuntimeRecoveryError({
290 operation: "reconcile",
291 threadId: projection.thread.id,
292 cause,
293 }),
294 ),
295 );
296 const hasReplayableCheckpoint = checkpointEffects.some(
297 (effect) =>
298 effect.request.type === "checkpoint.capture" &&
299 effect.request.runId === run.id &&
300 (effect.status === "pending" || effect.status === "running"),
301 );
302 if (hasReplayableCheckpoint) continue;
303 }
304 runs.push(run);
305 }
306 const pendingRequests = projection.runtimeRequests.filter(
307 (request) => request.status === "pending",
308 );
309 // The lost provider callback cannot resume; persisted questions can be
310 // answered by a new user message in threads that accept messages.
311 const questions = pendingRequests.filter(
312 (request) =>
313 request.kind === "user_input" &&
314 request.responseCapability.type === "live" &&
315 projection.turnItems.some(
316 (item) =>
317 item.type === "user_input_request" &&
318 item.requestId === request.id &&
319 item.nodeId === request.nodeId &&
320 item.questions.length > 0 &&
321 item.questions.every((question) => question.question.trim().length > 0),
322 ) &&
323 !isProviderNativeSubagentThread(projection.thread),
324 );
325 const recoveredRequestIds = new Set(questions.map((request) => request.id));
326 const messageRequestNodeIds = new Set(
327 pendingRequests
328 .filter(
329 (request) =>
330 request.responseCapability.type === "message" || recoveredRequestIds.has(request.id),
331 )
332 .map((request) => request.nodeId),
333 );
334 const requests = pendingRequests.filter(
335 (request) =>
336 request.responseCapability.type !== "message" && !recoveredRequestIds.has(request.id),
337 );
338 // Delegated task rows, items and nodes stay open: the child settles them.
339 const delegatedTaskNodeIds = new Set<string>([
340 ...(projection.subagents ?? []).filter(isAppOwnedDelegation).map((subagent) => subagent.id),
341 ...(projection.turnItems ?? []).flatMap((item) =>
342 item.type === "subagent" && isAppOwnedDelegation(item) ? [item.subagentId] : [],
343 ),
344 ]);
345 const detail = `Cancelled because the server ${trigger === "startup" ? "restarted" : "shut down"} before the provider work completed.`;
346 const commandId = CommandId.make(
347 `command:runtime-reconcile:${trigger}:${projection.thread.id}:${DateTime.formatIso(now)}`,
348 );
349 const allocateEventId = () =>
350 ids.allocate.event({ threadId: projection.thread.id, commandId }).pipe(
351 Effect.mapError(
352 (cause) =>
353 new ProviderRuntimeRecoveryError({
354 operation: "reconcile",
355 threadId: projection.thread.id,
356 cause,
357 }),
358 ),
359 );
360 const events: Array<OrchestrationV2DomainEvent> = [];
361 const cancelledBackgroundWork = new Map<
362 ProviderThreadId,
363 Array<OrchestrationV2RestartCancelledBackgroundWork>
364 >();
365 const cancelledBackgroundNativeIds = new Set<string>();
366 const recordCancelledBackgroundWork = (
367 providerThreadId: ProviderThreadId | null | undefined,
368 work: OrchestrationV2RestartCancelledBackgroundWork,
369 ) => {
370 if (providerThreadId == null) return;
371 const existing = cancelledBackgroundWork.get(providerThreadId);
372 if (existing === undefined) cancelledBackgroundWork.set(providerThreadId, [work]);
373 else existing.push(work);
374 };
375 const recordCancelledBackgroundItem = (
376 item: OrchestrationV2ThreadProjection["turnItems"][number],
377 ) => {
378 if (!isBackgroundCapableTurnItemType(item.type)) return;
379 const awaitingId = awaitingProviderThreadId(item, projection.runs);
380 const ownProviderThread =
381 item.providerThreadId === awaitingId
382 ? undefined
383 : (projection.providerThreads ?? []).find(
384 (candidate) => candidate.id === item.providerThreadId,
385 );
386 const work = cancelledTurnItemWork(
387 item,
388 ownProviderThread?.nativeThreadRef?.nativeId ?? undefined,
389 );
390 if (work === undefined) return;
391 recordCancelledBackgroundWork(awaitingId, work);
392 if (item.nativeItemRef?.nativeId != null) {
393 cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);
394 }
395 };
396 const unstartedRuns = runs.filter(
397 (run) =>
398 run.status === "starting" &&
399 neverStarted(run) &&
400 !projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) &&
401 !projection.runtimeRequests.some(
402 (request) =>
403 request.status === "pending" &&
404 projection.nodes.some((node) => node.id === request.nodeId && node.runId === run.id),
405 ),
406 );
407 const queuedRuns = projection.runs.filter((run) => run.status === "queued");
408 const pendingRuns = [...unstartedRuns, ...queuedRuns];
409 const queuedMessages =
410 pendingRuns.length === 0
411 ? []
412 : [
413 ...(yield* projections
414 .getMessagesById(
415 projection.thread.id,
416 pendingRuns.map((run) => run.userMessageId),
417 )
418 .pipe(
419 Effect.mapError(
420 (cause) =>
421 new ProviderRuntimeRecoveryError({
422 operation: "read-projections",
423 threadId: projection.thread.id,
424 cause,
425 }),
426 ),
427 )).values(),
428 ];
429 const bufferedMessageIds = new Set(
430 queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
431 );
432 const retryableMessageIds = new Set(
433 queuedMessages
434 .filter(
435 (message) =>
436 message.createdBy === "user" &&
437 message.scheduledTaskId === undefined &&
438 message.notification === undefined &&
439 message.delegatedCompletion === undefined,
440 )
441 .map((message) => message.id),
442 );
443 // A saved user message the provider never accepted stays available through the held queue.
444 const retryableRuns = unstartedRuns.filter((run) =>
445 retryableMessageIds.has(run.userMessageId),
446 );
447 // Nobody reviews a delegated, agent-sent or scheduled prompt, so it is sent again instead.
448 const unattendedRuns = continueAfterRestart
449 ? unstartedRuns.filter((run) =>
450 queuedMessages.some(
451 (message) =>
452 message.id === run.userMessageId &&
453 !retryableMessageIds.has(message.id) &&
454 message.notification === undefined &&
455 message.delegatedCompletion === undefined &&
456 (message.scheduledTaskId !== undefined || message.creationSource === "mcp"),
457 ),
458 )
459 : [];
460 const resentRuns: Array<OrchestrationV2ThreadProjection["runs"][number]> = [];
461 const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
462 for (const run of unattendedRuns) {
463 if (
464 restartContinuationBlocked(projection, run, now, false) ||
465 (yield* restartStopRequested(projections.getThreadRecords, projection.thread, run).pipe(
466 Effect.mapError(
467 (cause) =>
468 new ProviderRuntimeRecoveryError({
469 operation: "read-projections",
470 threadId: projection.thread.id,
471 cause,
472 }),
473 ),
474 ))
475 )
476 continue;
477 const resendCommandId = CommandId.make(`command:restart-resend:${run.id}`);
478 const earlierResends = yield* outbox.listByCommandId(resendCommandId).pipe(
479 Effect.mapError(
480 (cause) =>
481 new ProviderRuntimeRecoveryError({
482 operation: "reconcile",
483 threadId: projection.thread.id,
484 cause,
485 }),
486 ),
487 );
488 // A prompt whose start keeps crashing the server would otherwise be sent forever.
489 if (earlierResends.filter((effect) => effect.attemptCount > 0).length >= MAX_RESENDS) {
490 yield* Effect.logWarning("Not resending a prompt that restarts cut too many times", {
491 threadId: projection.thread.id,
492 runId: run.id,
493 });
494 continue;
495 }
496 resentRuns.push(run);
497 resends.push({
498 id: `effect:restart-resend:${run.id}:${commandId}`,
499 commandId: resendCommandId,
500 threadId: projection.thread.id,
501 request: { type: "provider-turn.start", runId: run.id },
502 });
503 }
504 for (const run of resentRuns) {
505 events.push({
506 id: yield* allocateEventId(),
507 type: "run.updated",
508 threadId: projection.thread.id,
509 runId: run.id,
510 providerInstanceId: run.providerInstanceId,
511 occurredAt: now,
512 payload: run,
513 });
514 }
515 const keptRuns = [...retryableRuns, ...resentRuns];
516 for (const run of keptRuns) {
517 for (const attempt of projection.attempts.filter(
518 (candidate) => candidate.id === run.activeAttemptId && candidate.status === "running",
519 )) {
520 events.push({
521 id: yield* allocateEventId(),
522 type: "run-attempt.updated",
523 threadId: projection.thread.id,
524 runId: run.id,
525 nodeId: attempt.rootNodeId,
526 providerInstanceId: run.providerInstanceId,
527 occurredAt: now,
528 payload: { ...attempt, status: "pending", startedAt: null },
529 });
530 }
531 for (const node of projection.nodes.filter(
532 (candidate) => candidate.id === run.rootNodeId && candidate.status === "running",
533 )) {
534 events.push({
535 id: yield* allocateEventId(),
536 type: "node.updated",
537 threadId: projection.thread.id,
538 runId: run.id,
539 nodeId: node.id,
540 providerInstanceId: run.providerInstanceId,
541 occurredAt: now,
542 payload: { ...node, status: "pending", startedAt: null },
543 });
544 }
545 }
546 const retryableItems =
547 retryableRuns.length === 0
548 ? []
549 : (yield* projections
550 .getThreadRecords(projection.thread.id, ["turnItems"], {
551 turnItemTypes: ["user_message"],
552 turnItemRunIds: retryableRuns.map((run) => run.id),
553 })
554 .pipe(
555 Effect.mapError(
556 (cause) =>
557 new ProviderRuntimeRecoveryError({
558 operation: "read-projections",
559 threadId: projection.thread.id,
560 cause,
561 }),
562 ),
563 )).turnItems;
564 for (const item of retryableItems) {
565 if (item.type !== "user_message" || !retryableMessageIds.has(item.messageId)) continue;
566 events.push({
567 id: yield* allocateEventId(),
568 type: "turn-item.updated",
569 threadId: projection.thread.id,
570 ...(item.runId === null ? {} : { runId: item.runId }),
571 occurredAt: now,
572 payload: {
573 ...item,
574 inputIntent: "queued_turn",
575 status: "pending",
576 startedAt: null,
577 completedAt: null,
578 updatedAt: now,
579 },
580 });
581 }
582 runs = runs.filter((run) => !keptRuns.some((kept) => kept.id === run.id));
583 const heldRuns = [
584 ...retryableRuns,
585 ...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
586 ];
587 for (const [index, run] of heldRuns.entries()) {
588 if (bufferedMessageIds.has(run.userMessageId)) {
589 runs.push(run);
590 continue;
591 }
592 if (run.queueHeld === true && retryableRuns.length === 0) continue;
593 events.push({
594 id: yield* allocateEventId(),
595 type: "run.updated",
596 threadId: projection.thread.id,
597 runId: run.id,
598 providerInstanceId: run.providerInstanceId,
599 occurredAt: now,
600 payload: {
601 ...run,
602 status: "queued",
603 queueHeld: true,
604 ...(retryableRuns.length === 0 ? {} : { queuePosition: index + 1 }),
605 },
606 });
607 }
608 const unsentResumeRunIds = runs.flatMap((run) =>
609 run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],
610 );
611 const limitRecovery = projection.thread.limitRecovery;
612 // The unsent resume spent the worker's request id, so a fresh one lets it send again. An
613 // untagged resume from an older server stays cancelled: its copy couldn't be hidden.
614 if (limitRecovery?.autoResume === true && unsentResumeRunIds.length > 0) {
615 events.push({
616 id: yield* allocateEventId(),
617 type: "thread.metadata-updated",
618 threadId: projection.thread.id,
619 occurredAt: now,
620 payload: {
621 ...projection.thread,
622 limitRecovery: { ...limitRecovery, requestId: commandId },
623 updatedAt: now,
624 },
625 });
626 }
627 // The worker sends the auto-resume again, so the unsent copy leaves the transcript the way
628 // cancelled queued input does.
629 const unsentResumePrompts =
630 unsentResumeRunIds.length === 0
631 ? []
632 : (yield* projections
633 .getThreadRecords(projection.thread.id, ["turnItems"], {
634 turnItemTypes: ["user_message"],
635 turnItemRunIds: unsentResumeRunIds,
636 })
637 .pipe(
638 Effect.mapError(
639 (cause) =>
640 new ProviderRuntimeRecoveryError({
641 operation: "read-projections",
642 threadId: projection.thread.id,
643 cause,
644 }),
645 ),
646 )).turnItems;
647 const unsentResumePromptIds = new Set(unsentResumePrompts.map((item) => item.id));
648 for (const item of unsentResumePrompts) {
649 if (item.type !== "user_message") continue;
650 const active =
651 item.status === "pending" || item.status === "running" || item.status === "waiting";
652 events.push({
653 id: yield* allocateEventId(),
654 type: "turn-item.updated",
655 threadId: projection.thread.id,
656 ...(item.runId === null ? {} : { runId: item.runId }),
657 occurredAt: now,
658 payload: {
659 ...item,
660 ...(active ? { status: "cancelled", completedAt: now } : {}),
661 inputIntent: "queued_turn",
662 updatedAt: now,
663 },
664 });
665 }
666 for (const question of questions) {
667 events.push({
668 id: yield* allocateEventId(),
669 type: "runtime-request.updated",
670 threadId: projection.thread.id,
671 nodeId: question.nodeId,
672 occurredAt: now,
673 payload: { ...question, responseCapability: { type: "message" } },
674 });
675 }
676 for (const request of requests) {
677 events.push({
678 id: yield* allocateEventId(),
679 type: "runtime-request.updated",
680 threadId: projection.thread.id,
681 nodeId: request.nodeId,
682 occurredAt: now,
683 payload: {
684 ...request,
685 status: trigger === "startup" ? "expired" : "cancelled",
686 responseCapability: {
687 type: "not_resumable",
688 reason: `The server ${trigger === "startup" ? "restarted" : "shut down"} before this runtime request was resolved.`,
689 },
690 resolvedAt: now,
691 },
692 });
693 }
694 for (const run of runs) {
695 events.push({
696 id: yield* allocateEventId(),
697 type: "run.updated",
698 threadId: projection.thread.id,
699 runId: run.id,
700 providerInstanceId: run.providerInstanceId,
701 occurredAt: now,
702 payload: { ...run, status: "cancelled", queuePosition: null, completedAt: now },
703 });
704 for (const attempt of projection.attempts.filter(
705 (candidate) =>
706 candidate.runId === run.id &&
707 (candidate.status === "pending" || candidate.status === "running"),
708 )) {
709 events.push({
710 id: yield* allocateEventId(),
711 type: "run-attempt.updated",
712 threadId: projection.thread.id,
713 runId: run.id,
714 nodeId: attempt.rootNodeId,
715 providerInstanceId: run.providerInstanceId,
716 occurredAt: now,
717 payload: { ...attempt, status: "cancelled", completedAt: now },
718 });
719 }
720 for (const node of projection.nodes.filter(
721 (candidate) =>
722 candidate.runId === run.id &&
723 !messageRequestNodeIds.has(candidate.id) &&
724 !delegatedTaskNodeIds.has(candidate.id) &&
725 (candidate.status === "pending" ||
726 candidate.status === "running" ||
727 candidate.status === "waiting"),
728 )) {
729 events.push({
730 id: yield* allocateEventId(),
731 type: "node.updated",
732 threadId: projection.thread.id,
733 runId: run.id,
734 nodeId: node.id,
735 providerInstanceId: run.providerInstanceId,
736 occurredAt: now,
737 payload: { ...node, status: "cancelled", completedAt: now },
738 });
739 }
740 for (const providerTurn of projection.providerTurns.filter(
741 (candidate) =>
742 candidate.runAttemptId !== null &&
743 projection.attempts.some(
744 (attempt) => attempt.id === candidate.runAttemptId && attempt.runId === run.id,
745 ) &&
746 (candidate.status === "pending" || candidate.status === "running"),
747 )) {
748 events.push({
749 id: yield* allocateEventId(),
750 type: "provider-turn.updated",
751 threadId: projection.thread.id,
752 runId: run.id,
753 nodeId: providerTurn.nodeId,
754 providerInstanceId: run.providerInstanceId,
755 occurredAt: now,
756 payload: { ...providerTurn, status: "cancelled", completedAt: now },
757 });
758 }
759 for (const message of projection.messages.filter(
760 (candidate) => candidate.runId === run.id && candidate.streaming,
761 )) {
762 events.push({
763 id: yield* allocateEventId(),
764 type: "message.updated",
765 threadId: projection.thread.id,
766 runId: run.id,
767 ...(message.nodeId === null ? {} : { nodeId: message.nodeId }),
768 providerInstanceId: run.providerInstanceId,
769 occurredAt: now,
770 payload: { ...message, streaming: false, updatedAt: now },
771 });
772 }
773 for (const item of projection.turnItems.filter(
774 (candidate) =>
775 candidate.runId === run.id &&
776 !unsentResumePromptIds.has(candidate.id) &&
777 (candidate.nodeId === null || !messageRequestNodeIds.has(candidate.nodeId)) &&
778 !isAppOwnedDelegationItem(candidate) &&
779 // A link card waits on the user, not the run, and answers by message.
780 candidate.type !== "link_request" &&
781 // The orchestrator queues a steer the provider never reported reading.
782 !(
783 candidate.type === "user_message" &&
784 candidate.status === "pending" &&
785 (candidate.inputIntent === "steer" ||
786 candidate.inputIntent === "promoted_queued_to_steer")
787 ) &&
788 (candidate.status === "pending" ||
789 candidate.status === "running" ||
790 candidate.status === "waiting"),
791 )) {
792 // Cut native subagents do not resume; record them for the continuation to restart.
793 if (run.status === "waiting") recordCancelledBackgroundItem(item);
794 else if (item.type === "subagent" && item.origin === "provider_native")
795 recordCancelledBackgroundItem(item);
796 events.push({
797 id: yield* allocateEventId(),
798 type: "turn-item.updated",
799 threadId: projection.thread.id,
800 runId: run.id,
801 ...(item.nodeId === null ? {} : { nodeId: item.nodeId }),
802 providerInstanceId: run.providerInstanceId,
803 occurredAt: now,
804 payload: cancelledItem(item, now),
805 });
806 }
807 // Queued runs and unsent resumes leave no visible turn behind, so there is nothing to explain.
808 if (run.status === "queued" || unsentResumeRunIds.includes(run.id)) continue;
809 events.push({
810 id: yield* allocateEventId(),
811 type: "turn-item.updated",
812 threadId: projection.thread.id,
813 runId: run.id,
814 ...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
815 providerInstanceId: run.providerInstanceId,
816 occurredAt: now,
817 payload: {
818 id: ids.derive.runSignalTurnItem({ runId: run.id, signal: "server-recovery" }),
819 threadId: projection.thread.id,
820 runId: run.id,
821 nodeId: run.rootNodeId,
822 providerThreadId: run.providerThreadId,
823 providerTurnId: null,
824 nativeItemRef: null,
825 parentItemId: null,
826 // EventSink allocates the real position, after the run's existing items.
827 ordinal: 0,
828 status: "completed",
829 title: trigger === "startup" ? "Server restarted" : "Server shut down",
830 startedAt: now,
831 completedAt: now,
832 updatedAt: now,
833 type: "system_notice",
834 message: detail,
835 },
836 });
837 }
838 // Blocking waits die with their turn; let child results wake the parent continuation as async tasks.
839 const cutRunIds = new Set(runs.map((run) => run.id));
840 for (const task of projection.subagents ?? []) {
841 if (
842 task.origin !== "app_owned" ||
843 task.runId === null ||
844 !cutRunIds.has(task.runId) ||
845 task.completionWake !== "settled_only" ||
846 !isOrchestrationV2WorkActive(task.status)
847 )
848 continue;
849 events.push({
850 id: yield* allocateEventId(),
851 type: "subagent.updated",
852 threadId: projection.thread.id,
853 runId: task.runId,
854 nodeId: task.id,
855 driver: task.driver,
856 providerInstanceId: task.providerInstanceId,
857 occurredAt: now,
858 payload: { ...task, completionWake: "always", updatedAt: now },
859 });
860 }
861 const recoveredNonterminalRunIds = new Set(runs.map((run) => run.id));
862 const cancelledStaleNodeIds = new Set<string>();
863 for (const item of projection.turnItems ?? []) {
864 if (item.runId !== null && recoveredNonterminalRunIds.has(item.runId)) {
865 continue;
866 }
867 if (!isBackgroundCapableTurnItemType(item.type)) {
868 continue;
869 }
870 if (!isNonterminalTurnItemStatus(item.status) || isAppOwnedDelegationItem(item)) {
871 continue;
872 }
873 const providerInstanceId = resolveStaleBackgroundItemProviderInstanceId(item, projection);
874 recordCancelledBackgroundItem(item);
875 events.push({
876 id: yield* allocateEventId(),
877 type: "turn-item.updated",
878 threadId: projection.thread.id,
879 ...(item.runId === null ? {} : { runId: item.runId }),
880 ...(item.nodeId === null || item.nodeId === undefined ? {} : { nodeId: item.nodeId }),
881 providerInstanceId,
882 occurredAt: now,
883 payload: cancelledItem(item, now),
884 });
885 if (item.nodeId !== null && item.nodeId !== undefined) {
886 const staleItemNode = projection.nodes.find(
887 (candidate) =>
888 candidate.id === item.nodeId && isNonterminalTurnItemStatus(candidate.status),
889 );
890 if (staleItemNode !== undefined && !cancelledStaleNodeIds.has(staleItemNode.id)) {
891 cancelledStaleNodeIds.add(staleItemNode.id);
892 events.push({
893 id: yield* allocateEventId(),
894 type: "node.updated",
895 threadId: projection.thread.id,
896 ...(item.runId === null ? {} : { runId: item.runId }),
897 nodeId: staleItemNode.id,
898 providerInstanceId,
899 occurredAt: now,
900 payload: { ...staleItemNode, status: "cancelled", completedAt: now },
901 });
902 }
903 }
904 }
905 // No provider process survives the server; open native subagents can never report completion.
906 for (const subagent of projection.subagents ?? []) {
907 events.push(
908 ...(yield* endOrphanedNativeSubagent({
909 projection,
910 subagentId: subagent.id,
911 status: "cancelled",
912 now,
913 emitted: events,
914 allocateEventId,
915 })),
916 );
917 }
918 // Native subagent threads have no runs to settle their runless root turns.
919 events.push(
920 ...(yield* endRunlessRootTurns({
921 threadId: projection.thread.id,
922 providerInstanceId: projection.thread.providerInstanceId,
923 projection,
924 status: "cancelled",
925 now,
926 emitted: events,
927 allocateEventId,
928 })),
929 );
930 for (const providerThread of projection.providerThreads ?? []) {
931 const needsIdle = providerThread.status === "active";
932 const needsRosterClear = providerThreadHasPendingBackgroundTasks(providerThread);
933 if (!needsIdle && !needsRosterClear) {
934 continue;
935 }
936 if (providerThread.ownerNodeId === null) {
937 for (const task of providerThread.pendingBackgroundTasks ?? []) {
938 // A subagent already recorded covers the shells it started.
939 if (
940 cancelledBackgroundNativeIds.has(task.taskId) ||
941 (task.parentTaskId !== undefined &&
942 cancelledBackgroundNativeIds.has(task.parentTaskId))
943 )
944 continue;
945 cancelledBackgroundNativeIds.add(task.taskId);
946 recordCancelledBackgroundWork(providerThread.id, cancelledRosterTaskWork(task));
947 }
948 }
949 events.push({
950 id: yield* allocateEventId(),
951 type: "provider-thread.updated",
952 threadId: projection.thread.id,
953 driver: providerThread.driver,
954 providerInstanceId: providerThread.providerInstanceId,
955 occurredAt: now,
956 payload: {
957 ...providerThread,
958 status: needsIdle ? "idle" : providerThread.status,
959 pendingBackgroundTasks: [],
960 updatedAt: now,
961 },
962 });
963 }
964 for (const session of projection.providerSessions.filter(
965 (candidate) =>
966 (candidate.status !== "stopped" && candidate.status !== "error") ||
967 (candidate.status === "error" && (candidate.extensionStatuses?.length ?? 0) > 0),
968 )) {
969 events.push({
970 id: yield* allocateEventId(),
971 type: "provider-session.updated",
972 threadId: projection.thread.id,
973 driver: session.driver,
974 providerInstanceId: session.providerInstanceId,
975 occurredAt: now,
976 payload: {
977 ...session,
978 status: session.status === "error" ? "error" : "stopped",
979 ...(session.extensionStatuses === undefined ? {} : { extensionStatuses: [] }),
980 updatedAt: now,
981 lastError: session.status === "error" ? session.lastError : null,
982 },
983 });
984 }
985 for (const [providerThreadId, work] of cancelledBackgroundWork) {
986 const noteRun = latestStartedRun(projection, providerThreadId);
987 if (noteRun === undefined) continue;
988 // Use a separate event; stale run snapshots can overwrite intervening lifecycle changes.
989 events.push({
990 id: yield* allocateEventId(),
991 type: "run.background-work-cancelled",
992 threadId: projection.thread.id,
993 runId: noteRun.id,
994 providerInstanceId: noteRun.providerInstanceId,
995 occurredAt: now,
996 payload: {
997 runId: noteRun.id,
998 restartCancelledBackgroundWork: mergeRestartCancelledBackgroundWork(
999 noteRun.restartCancelledBackgroundWork ?? [],
1000 work,
1001 ),
1002 },
1003 });
1004 }
1005 const continuationRun =
1006 continueAfterRestart && trigger === "startup"
1007 ? restartContinuationRun(
1008 { ...projection, providerSessions: sessionsLiveAtStart(projection, stoppedThisPass) },
1009 now,
1010 )
1011 : undefined;
1012 let processLossCount = 1;
1013 // Normal upgrades must not exhaust an unfinished turn's crash allowance.
1014 if (
1015 continuationRun?.restartContinuationOfRunId !== undefined &&
1016 continuationRun.status !== "completed" &&
1017 continuationRun.status !== "waiting"
1018 ) {
1019 const previous = yield* outbox
1020 .get(`effect:restart-continuation:${continuationRun.restartContinuationOfRunId}`)
1021 .pipe(
1022 Effect.mapError(
1023 (cause) =>
1024 new ProviderRuntimeRecoveryError({
1025 operation: "reconcile",
1026 threadId: projection.thread.id,
1027 cause,
1028 }),
1029 ),
1030 );
1031 if (Option.isSome(previous) && previous.value.request.type === "provider-runtime.continue")
1032 processLossCount += previous.value.request.processLossCount ?? 0;
1033 }
1034 const effects: Array<EffectOutbox.PendingOrchestrationEffectV2> = continuationRun
1035 ? [
1036 {
1037 id: `effect:restart-continuation:${continuationRun.id}`,
1038 commandId,
1039 threadId: projection.thread.id,
1040 request: {
1041 type: "provider-runtime.continue",
1042 sourceRunId: continuationRun.id,
1043 processLossCount,
1044 },
1045 },
1046 ]
1047 : [];
1048 const stoppedSessions = projection.providerSessions.filter(
1049 (candidate) => candidate.status !== "stopped" && candidate.status !== "error",
1050 ).length;
1051 let retiredEffects: number;
1052 let cancelledEffectIds: ReadonlyArray<string> = [];
1053 if (events.length === 0) {
1054 const retiredEffectIds = yield* outbox
1055 .cancelUnsettled({
1056 threadId: projection.thread.id,
1057 effectTypes: EffectOutbox.PROCESS_BOUND_EFFECT_TYPES,
1058 reason: detail,
1059 })
1060 .pipe(
1061 Effect.mapError(
1062 (cause) =>
1063 new ProviderRuntimeRecoveryError({
1064 operation: "reconcile",
1065 threadId: projection.thread.id,
1066 cause: { detail, cause },
1067 }),
1068 ),
1069 );
1070 cancelledEffectIds = retiredEffectIds;
1071 retiredEffects = retiredEffectIds.length;
1072 } else {
1073 const result = yield* eventSink
1074 .commitCommand({
1075 commandId,
1076 threadId: projection.thread.id,
1077 commandType: "provider-runtime.reconcile",
1078 acceptedAt: now,
1079 events,
1080 effects,
1081 cancelUnsettledEffects: {
1082 effectTypes: EffectOutbox.PROCESS_BOUND_EFFECT_TYPES,
1083 reason: detail,
1084 },
1085 })
1086 .pipe(
1087 Effect.mapError(
1088 (cause) =>
1089 new ProviderRuntimeRecoveryError({
1090 operation: "reconcile",
1091 threadId: projection.thread.id,
1092 cause,
1093 }),
1094 ),
1095 );
1096 retiredEffects = result.cancelledEffectCount;
1097 }
1098 return {
1099 terminalizedRuns: runs.length,
1100 stoppedSessions,
1101 closedRequests: requests.length,
1102 retiredEffects,
1103 cancelledEffectIds,
1104 resends,
1105 };
1106 },
1107 );
1108
1109 const reconcile = Effect.fn("ProviderRuntimeRecoveryService.reconcile")(function* (
1110 trigger: "startup" | "shutdown",
1111 ) {
1112 const continueAfterRestart = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
1113 const threadIds = yield* projections
1114 .getRecoveryThreadIds("runtime")
1115 .pipe(
1116 Effect.mapError(
1117 (cause) => new ProviderRuntimeRecoveryError({ operation: "read-projections", cause }),
1118 ),
1119 );
1120 let terminalizedRuns = 0;
1121 let stoppedSessions = 0;
1122 let closedRequests = 0;
1123 let retiredEffects = 0;
1124 const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
1125 const stoppedThisPass = new Map<string, RecoverySession>();
1126 const grouped = trigger === "shutdown" && threadIds.length > 1;
1127 const groupSize = grouped ? 32 : 1;
1128 for (let offset = 0; offset < threadIds.length; offset += groupSize) {
1129 const cancelledEffectIds: string[] = [];
1130 // Read and decide on the writer so shared sessions see earlier updates in this group.
1131 const reconcileGroup = Effect.gen(function* () {
1132 for (const threadId of threadIds.slice(offset, offset + groupSize)) {
1133 const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
1134 Effect.mapError(
1135 (cause) =>
1136 new ProviderRuntimeRecoveryError({
1137 operation: "read-projections",
1138 threadId,
1139 cause,
1140 }),
1141 ),
1142 );
1143 const enabled =
1144 continueAfterRestart !== null &&
1145 resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
1146 .continueThreadsAfterServerUpdate;
1147 const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
1148 for (const session of projection.providerSessions) {
1149 if (session.status !== "stopped" && session.status !== "error") {
1150 stoppedThisPass.set(session.id, session);
1151 }
1152 }
1153 cancelledEffectIds.push(...result.cancelledEffectIds);
1154 resends.push(...result.resends);
1155 terminalizedRuns += result.terminalizedRuns;
1156 stoppedSessions += result.stoppedSessions;
1157 closedRequests += result.closedRequests;
1158 retiredEffects += result.retiredEffects;
1159 }
1160 });
1161 yield* (grouped ? eventSink.withProviderBatch(reconcileGroup) : reconcileGroup).pipe(
1162 Effect.mapError((cause) =>
1163 isProviderRuntimeRecoveryError(cause)
1164 ? cause
1165 : new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
1166 ),
1167 );
1168 // Eventless outbox cancellations also wait for the group's durable commit.
1169 if (cancelledEffectIds.length > 0) yield* outbox.signalCancellations(cancelledEffectIds);
1170 }
1171 const outboxReconciliation = yield* outbox.reconcileAfterProcessLoss.pipe(
1172 Effect.mapError(
1173 (cause) => new ProviderRuntimeRecoveryError({ operation: "drain-outbox", cause }),
1174 ),
1175 );
1176 // After the drain, which cancels every unsettled provider start. A crash before this write
1177 // leaves the run unstarted, so the next recovery sends it again.
1178 if (resends.length > 0) {
1179 yield* eventSink
1180 .writeWithEffects({ events: [], effects: resends })
1181 .pipe(
1182 Effect.mapError(
1183 (cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
1184 ),
1185 );
1186 }
1187 return {
1188 terminalizedRuns,
1189 stoppedSessions,
1190 closedRequests,
1191 retiredEffects: retiredEffects + outboxReconciliation.cancelled,
1192 requeuedEffects: outboxReconciliation.requeued,
1193 } satisfies ProviderRuntimeReconciliationSummary;
1194 });
1195
1196 const prepareForShutdown = Effect.gen(function* () {
1197 const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
1198 if (!enabled) return;
1199 const enabledProjects = Object.entries(enabled.projectSettingsOverrides)
1200 .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === true)
1201 .map(([id]) => id);
1202 if (!enabled.continueThreadsAfterServerUpdate && enabledProjects.length === 0) return;
1203 const disabledProjects = Object.entries(enabled.projectSettingsOverrides)
1204 .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === false)
1205 .map(([id]) => id);
1206 const threadIds = yield* projections.getRecoveryThreadIds(
1207 "restart-continuations",
1208 enabled.continueThreadsAfterServerUpdate
1209 ? { exclude: disabledProjects }
1210 : { include: enabledProjects },
1211 );
1212 yield* Effect.forEach(
1213 threadIds,
1214 (threadId) =>
1215 Effect.gen(function* () {
1216 const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
1217 if (
1218 !resolveProjectSettings(enabled, projection.thread.projectId).settings
1219 .continueThreadsAfterServerUpdate
1220 )
1221 return;
1222 const run = restartContinuationRun(
1223 yield* rewindUnacceptedRuns(projection),
1224 yield* DateTime.now,
1225 );
1226 if (!run) return;
1227 const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
1228 yield* eventSink.writeWithEffects({
1229 commandId,
1230 events: [],
1231 effects: [
1232 {
1233 id: `effect:restart-continuation:${run.id}`,
1234 commandId,
1235 threadId,
1236 request: {
1237 type: "provider-runtime.continue",
1238 sourceRunId: run.id,
1239 processLossCount: 0,
1240 },
1241 },
1242 ],
1243 });
1244 }).pipe(
1245 Effect.catchCauseIf(
1246 (cause) => !Cause.hasInterruptsOnly(cause),
1247 (cause) =>
1248 Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),
1249 ),
1250 ),
1251 // Leave room in the four-reader pool for other shutdown reads.
1252 { concurrency: 2, discard: true },
1253 );
1254 }).pipe(
1255 Effect.withSpan("ProviderRuntimeRecoveryService.prepareForShutdown"),
1256 Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })),
1257 );
1258
1259 const recover = Effect.gen(function* () {
1260 return (yield* reconcile("startup")) satisfies ProviderRuntimeRecoverySummary;
1261 });
1262
1263 return ProviderRuntimeRecoveryService.of({ reconcile, prepareForShutdown, recover });
1264 });
1265
1266 export const layer = Layer.effect(ProviderRuntimeRecoveryService, make);