@devries-cameron20 / t3code Mirror · read-only / commits / 34204a39e8
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 12:15 UTC

fix(server): restarts no longer drop prompts the provider never accepted (#32413)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Jolymmiles committed Nov 15, 2026 at 08:32 UTC 34204a39e8702372edcabd129696e6acb9caca2c
8 files changed +671 -105
apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
+10 -3
@@ -13,6 +13,7 @@ import {
13 ProviderSessionId,
14 ProviderThreadId,
15 ProviderTurnId,
16 + RunAttemptId,
17 RunId,
18 RuntimeRequestId,
19 ThreadId,
@@ -1239,7 +1240,13 @@ it.effect("reads the run that owns a background roster, not a queued or resumed
1240 }).pipe(Effect.provide(layerTest)),
1241 );
1242
1242 -it.effect("wakes the run whose subagent lost its shell, past a wake cut before it started", () =>
1243 +it.effect.each([
1244 + ["a wake cut before it started", { status: "starting", startedAt: null }],
1245 + [
1246 + "a prompt the provider never accepted",
1247 + { status: "running", activeAttemptId: RunAttemptId.make("attempt:unaccepted") },
1248 + ],
1249 +] as const)("wakes the run whose subagent lost its shell, past %s", ([, later]) =>
1250 Effect.gen(function* () {
1251 const projections = yield* ProjectionStore.ProjectionStoreV2;
1252 const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
@@ -1269,11 +1276,11 @@ it.effect("wakes the run whose subagent lost its shell, past a wake cut before i
1276 },
1277 });
1278 const settledRun = yield* createRun(threadId, "completed", { providerThreadId });
1272 - yield* createRun(threadId, "starting", {
1279 + yield* createRun(threadId, later.status, {
1280 ordinal: 2,
1281 providerThreadId,
1275 - startedAt: null,
1282 completedAt: null,
1283 + ...later,
1284 });
1285 yield* projections.apply({
1286 id: EventId.make(`event:${threadId}:roster`),
apps/server/src/orchestration-v2/ProjectionStore.ts
+6 -3
@@ -5265,14 +5265,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5265 WHERE latest.thread_id = ${threadId}
5266 ORDER BY latest.ordinal DESC LIMIT 1
5267 )
5268 - -- Retain the latest run for each provider thread with lost
5269 - -- background work, including owners used before a handoff.
5268 + -- Retain the latest ended run for each provider thread with lost
5269 + -- background work, including owners used before a handoff. An
5270 + -- unfinished run is read above and may prove never started.
5271 OR run.run_id IN (
5272 SELECT (
5273 SELECT ended.run_id FROM orchestration_v2_projection_runs AS ended
5274 WHERE ended.thread_id = ${threadId}
5275 AND ended.provider_thread_id = roster.provider_thread_id
5275 - AND ended.status NOT IN ('queued', 'rolled_back')
5276 + AND ended.status NOT IN (
5277 + 'queued', 'preparing', 'starting', 'running', 'waiting', 'rolled_back'
5278 + )
5279 -- A run that never started (see neverStarted) owns no work.
5280 AND NOT (
5281 json_type(ended.payload_json, '$.startedAt') = 'null'
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
+156 -23
@@ -14,6 +14,7 @@ import {
14 TurnItemId,
15 type OrchestrationV2ThreadProjection,
16 } from "@t3tools/contracts";
17 +import * as DateTime from "effect/DateTime";
18 import * as Effect from "effect/Effect";
19 import * as Layer from "effect/Layer";
20
@@ -428,8 +429,17 @@ it.effect("recovery closes a secret card's form but leaves a link card open", ()
429 runtimeRequests: [],
430 providerSessions: [],
431 providerThreads: [],
431 - providerTurns: [],
432 - runs: [{ id: runId, status: "running", providerInstanceId: ProviderInstanceId.make("codex") }],
432 + providerTurns: [
433 + { runAttemptId: RunAttemptId.make("attempt_secret_recovery"), status: "running" },
434 + ],
435 + runs: [
436 + {
437 + id: runId,
438 + status: "running",
439 + activeAttemptId: RunAttemptId.make("attempt_secret_recovery"),
440 + providerInstanceId: ProviderInstanceId.make("codex"),
441 + },
442 + ],
443 attempts: [],
444 nodes: [],
445 subagents: [],
@@ -508,20 +518,62 @@ it.effect.each([
518 "starting-other-request",
519 "starting-own-request",
520 "starting-scheduled",
521 + "starting-steered",
522 + "running",
523 + "running-delegated",
524 + "running-delegated-capped",
525 + "running-delegated-snoozed",
526 + "running-delegated-stopped",
527 + "running-resume",
528 ] as const)("recovers $0 work after restart", (scenario) => {
512 - const status = scenario.startsWith("starting") ? "starting" : scenario;
529 + const status = scenario.startsWith("starting")
530 + ? "starting"
531 + : scenario.startsWith("running")
532 + ? "running"
533 + : scenario;
534 + const resent = scenario === "starting-scheduled" || scenario === "running-delegated";
535 const terminalized =
536 status === "preparing" ||
537 scenario === "starting-own-request" ||
516 - scenario === "starting-scheduled";
538 + scenario === "starting-steered" ||
539 + scenario === "running-delegated-capped" ||
540 + scenario === "running-delegated-snoozed" ||
541 + scenario === "running-delegated-stopped" ||
542 + scenario === "running-resume";
543 + const startedAt =
544 + status === "running" || scenario === "starting-steered"
545 + ? DateTime.makeUnsafe("2026-11-01T19:34:40.403Z")
546 + : null;
547 const threadId = ThreadId.make("thread_queued_restart");
548 const runId = RunId.make("run_queued_restart");
549 const attemptId = RunAttemptId.make("attempt_queued_restart");
550 + const steeredAttemptId = RunAttemptId.make("attempt_queued_restart_steered");
551 const rootNodeId = NodeId.make("node_queued_restart");
552 let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
553 null;
554 + const written: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
555 const projection = {
524 - thread: { id: threadId },
556 + thread: {
557 + id: threadId,
558 + archivedAt: null,
559 + deletedAt: null,
560 + settledOverride: null,
561 + snoozedUntil:
562 + scenario === "running-delegated-snoozed"
563 + ? DateTime.makeUnsafe("2099-07-25T09:00:00.000Z")
564 + : null,
565 + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
566 + forkedFrom: null,
567 + ...(scenario === "running-resume"
568 + ? {
569 + limitRecovery: {
570 + runId: RunId.make("run_queued_restart_limited"),
571 + resetAt: "2026-11-01T19:34:00.000Z",
572 + autoResume: true,
573 + },
574 + }
575 + : {}),
576 + },
577 runtimeRequests: scenario.endsWith("request")
578 ? [
579 {
@@ -536,16 +588,29 @@ it.effect.each([
588 : [],
589 providerSessions: [],
590 providerThreads: [],
539 - providerTurns: [],
591 + providerTurns:
592 + scenario === "starting-steered"
593 + ? [
594 + {
595 + id: ProviderTurnId.make("provider_turn_queued_restart_steered"),
596 + runAttemptId: steeredAttemptId,
597 + status: "interrupted",
598 + },
599 + ]
600 + : [],
601 runs: [
602 {
603 id: runId,
604 status,
544 - startedAt: null,
605 + startedAt,
606 + rootNodeId,
607 activeAttemptId: attemptId,
608 userMessageId: MessageId.make("message_queued_restart"),
609 queuePosition: 1,
610 providerInstanceId: ProviderInstanceId.make("codex"),
611 + ...(scenario === "running-resume"
612 + ? { usageLimitContinuationOfRunId: RunId.make("run_queued_restart_limited") }
613 + : {}),
614 ...(status === "preparing"
615 ? {
616 workspacePreparation: {
@@ -558,11 +623,15 @@ it.effect.each([
623 },
624 ],
625 attempts: [
626 + ...(scenario === "starting-steered"
627 + ? [{ id: steeredAttemptId, runId, rootNodeId, status: "interrupted" }]
628 + : []),
629 {
630 id: attemptId,
631 runId,
632 rootNodeId,
565 - status: "pending",
633 + status: status === "running" ? "running" : "pending",
634 + startedAt: status === "running" ? startedAt : null,
635 },
636 ],
637 nodes: [
@@ -578,21 +647,37 @@ it.effect.each([
647 {
648 id: rootNodeId,
649 runId,
581 - status: "pending",
650 + status: status === "running" ? "running" : "pending",
651 + startedAt: status === "running" ? startedAt : null,
652 },
653 ],
654 subagents: [],
655 messages:
586 - status === "starting"
656 + status === "starting" || status === "running"
657 ? [
658 {
659 id: MessageId.make("message_queued_restart"),
590 - createdBy: "user",
660 + ...(scenario.startsWith("running-delegated")
661 + ? { createdBy: "agent", creationSource: "mcp" }
662 + : scenario === "running-resume"
663 + ? { createdBy: "agent", creationSource: "server" }
664 + : { createdBy: "user" }),
665 ...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),
666 },
667 ]
668 : [],
595 - turnItems: [],
669 + turnItems:
670 + scenario === "running-delegated-stopped"
671 + ? [
672 + {
673 + id: TurnItemId.make("turn_item_queued_restart_stop"),
674 + type: "run_interrupt_request",
675 + runId,
676 + nodeId: rootNodeId,
677 + status: "completed",
678 + },
679 + ]
680 + : [],
681 } as unknown as OrchestrationV2ThreadProjection;
682 const layer = ProviderRuntimeRecovery.layer.pipe(
683 Layer.provide(ServerSettings.layerTest()),
@@ -600,7 +685,12 @@ it.effect.each([
685 Layer.mergeAll(
686 Layer.mock(ProjectionStore.ProjectionStoreV2)({
687 getRecoveryThreadIds: () => Effect.succeed([threadId]),
603 - getRuntimeRecoveryProjection: () => Effect.succeed(projection),
688 + // Like the store, recovery state carries only provider turns that have not ended.
689 + getRuntimeRecoveryProjection: () =>
690 + Effect.succeed({
691 + ...projection,
692 + providerTurns: projection.providerTurns.filter((turn) => turn.status === "running"),
693 + }),
694 getMessagesById: (requestedThreadId, messageIds) =>
695 Effect.succeed(
696 new Map(
@@ -618,12 +708,23 @@ it.effect.each([
708 committedInput = input;
709 return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
710 },
711 + writeWithEffects: (input) => {
712 + written.push(...input.effects);
713 + return Effect.succeed([]);
714 + },
715 }),
716 IdAllocator.layer,
717 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
718 runRecoveryOnce: Effect.succeed(false),
719 }),
720 Layer.mock(EffectOutbox.EffectOutboxV2)({
721 + // Three earlier restarts each cut a resend the worker had claimed.
722 + listByCommandId: () =>
723 + Effect.succeed(
724 + scenario === "running-delegated-capped"
725 + ? ([1, 1, 1].map((attemptCount) => ({ attemptCount })) as never)
726 + : [],
727 + ),
728 reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
729 }),
730 ),
@@ -640,11 +741,15 @@ it.effect.each([
741 const runEvent = command.events.find((event) => event.type === "run.updated");
742 const attemptEvent = command.events.find((event) => event.type === "run-attempt.updated");
743 const nodeEvent = command.events.find((event) => event.type === "node.updated");
643 - assert.equal(
644 - runEvent?.type === "run.updated" ? runEvent.payload.status : null,
645 - terminalized ? "cancelled" : "queued",
646 - );
744 + assert.deepInclude(runEvent?.type === "run.updated" ? runEvent.payload : null, {
745 + status: terminalized ? "cancelled" : resent ? "starting" : "queued",
746 + startedAt: scenario === "starting-steered" ? startedAt : null,
747 + });
748 assert.isEmpty(command.effects);
749 + assert.deepEqual(
750 + written.map((effect) => effect.request),
751 + resent ? [{ type: "provider-turn.start", runId }] : [],
752 + );
753 if (terminalized) {
754 assert.equal(
755 attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,
@@ -655,10 +760,33 @@ it.effect.each([
760 scenario === "starting-own-request" ? null : "cancelled",
761 );
762 } else {
658 - assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
659 - assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queueHeld : false, true);
660 - assert.isUndefined(attemptEvent);
661 - assert.isUndefined(nodeEvent);
763 + if (!resent) {
764 + assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
765 + assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queueHeld : false, true);
766 + }
767 + if (status === "running") {
768 + assert.deepInclude(
769 + attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload : null,
770 + { status: "pending", startedAt: null },
771 + );
772 + assert.deepInclude(nodeEvent?.type === "node.updated" ? nodeEvent.payload : null, {
773 + status: "pending",
774 + startedAt: null,
775 + });
776 + } else {
777 + assert.isUndefined(attemptEvent);
778 + assert.isUndefined(nodeEvent);
779 + }
780 + }
781 + // The unsent auto-resume gets a fresh request id so the limit worker sends it again.
782 + if (scenario === "running-resume") {
783 + const rearmed = command.events.find((event) => event.type === "thread.metadata-updated");
784 + assert.equal(
785 + rearmed?.type === "thread.metadata-updated"
786 + ? rearmed.payload.limitRecovery?.requestId
787 + : null,
788 + command.commandId,
789 + );
790 }
791 }).pipe(Effect.provide(layer));
792 });
@@ -707,6 +835,7 @@ it.effect(
835 {
836 id: runId,
837 status: "starting",
838 + activeAttemptId: attemptId,
839 providerThreadId,
840 providerInstanceId: ProviderInstanceId.make("codex"),
841 },
@@ -975,7 +1104,7 @@ it.effect(
1104 pendingBackgroundTasks: [],
1105 },
1106 ],
978 - providerTurns: [],
1107 + providerTurns: [{ runAttemptId: activeAttemptId, status: "running" }],
1108 runs: [
1109 {
1110 id: settledRunId,
@@ -985,6 +1114,7 @@ it.effect(
1114 {
1115 id: activeRunId,
1116 status: "running",
1117 + activeAttemptId,
1118 providerInstanceId: claudeInstanceId,
1119 },
1120 ],
@@ -1440,7 +1570,9 @@ it.effect("leaves delegated tasks to their own child threads after process loss"
1570 pendingBackgroundTasks: [],
1571 },
1572 ],
1443 - providerTurns: [],
1573 + providerTurns: [
1574 + { runAttemptId: RunAttemptId.make("attempt_recovery_delegation"), status: "running" },
1575 + ],
1576 runs: [
1577 {
1578 id: settledRunId,
@@ -1453,6 +1585,7 @@ it.effect("leaves delegated tasks to their own child threads after process loss"
1585 id: runningRunId,
1586 ordinal: 2,
1587 status: "running",
1588 + activeAttemptId: RunAttemptId.make("attempt_recovery_delegation"),
1589 providerThreadId,
1590 providerInstanceId: claudeInstanceId,
1591 },
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+184 -5
@@ -5,6 +5,7 @@ import {
5 isProviderNativeSubagentThread,
6 type OrchestrationV2DomainEvent,
7 type ProviderThreadId,
8 + type RunAttemptId,
9 type OrchestrationV2RestartCancelledBackgroundWork,
10 type OrchestrationV2Subagent,
11 type OrchestrationV2ThreadProjection,
@@ -24,7 +25,11 @@ import * as EventSink from "./EventSink.ts";
25 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
26 import * as ProjectionStore from "./ProjectionStore.ts";
27 import * as ServerSettings from "../serverSettings.ts";
27 -import { restartContinuationRun } from "./RestartContinuation.ts";
28 +import {
29 + restartContinuationBlocked,
30 + restartContinuationRun,
31 + restartStopRequested,
32 +} from "./RestartContinuation.ts";
33 import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
34 import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
35 import {
@@ -54,6 +59,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
59
60 const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);
61
62 +const MAX_RESENDS = 3;
63 +
64 export interface ProviderRuntimeRecoverySummary {
65 readonly terminalizedRuns: number;
66 readonly stoppedSessions: number;
@@ -206,14 +213,71 @@ export const make = Effect.gen(function* () {
213 return [...sessions.values()];
214 };
215
216 + /**
217 + * A run is marked running before its provider accepts the prompt, so only a provider turn of
218 + * one of its attempts proves the prompt arrived. Recovery treats the others as never started.
219 + */
220 + const rewindUnacceptedRuns = Effect.fn("ProviderRuntimeRecoveryService.rewindUnacceptedRuns")(
221 + function* (projection: ProjectionStore.ProjectionRuntimeRecoveryState) {
222 + const runIdByAttempt = new Map(
223 + projection.runs.flatMap((run) =>
224 + (run.status === "running" || (run.status === "starting" && run.startedAt !== null)) &&
225 + run.activeAttemptId !== null
226 + ? [[run.activeAttemptId, run.id] as const]
227 + : [],
228 + ),
229 + );
230 + if (runIdByAttempt.size === 0) return projection;
231 + const unaccepted = new Set(runIdByAttempt.values());
232 + const accept = (turns: ReadonlyArray<{ readonly runAttemptId: RunAttemptId | null }>) => {
233 + for (const turn of turns) {
234 + const runId =
235 + turn.runAttemptId === null ? undefined : runIdByAttempt.get(turn.runAttemptId);
236 + if (runId !== undefined) unaccepted.delete(runId);
237 + }
238 + };
239 + accept(projection.providerTurns);
240 + if (unaccepted.size === 0) return projection;
241 + for (const attempt of projection.attempts) {
242 + if (unaccepted.has(attempt.runId)) runIdByAttempt.set(attempt.id, attempt.runId);
243 + }
244 + // The recovery projection omits ended provider turns, such as one a steer replaced.
245 + accept(
246 + (yield* projections
247 + .getThreadRecords(projection.thread.id, ["providerTurns"], {
248 + providerTurnAttemptIds: [...runIdByAttempt].flatMap(([attemptId, runId]) =>
249 + unaccepted.has(runId) ? [attemptId] : [],
250 + ),
251 + })
252 + .pipe(
253 + Effect.mapError(
254 + (cause) =>
255 + new ProviderRuntimeRecoveryError({
256 + operation: "read-projections",
257 + threadId: projection.thread.id,
258 + cause,
259 + }),
260 + ),
261 + )).providerTurns,
262 + );
263 + return {
264 + ...projection,
265 + runs: projection.runs.map((run) =>
266 + unaccepted.has(run.id) ? { ...run, status: "starting" as const, startedAt: null } : run,
267 + ),
268 + };
269 + },
270 + );
271 +
272 const reconcileProjection = Effect.fn("ProviderRuntimeRecoveryService.reconcileProjection")(
273 function* (
211 - projection: ProjectionStore.ProjectionRuntimeRecoveryState,
274 + stored: ProjectionStore.ProjectionRuntimeRecoveryState,
275 trigger: "startup" | "shutdown",
276 continueAfterRestart: boolean,
277 stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
278 ) {
279 const now = yield* DateTime.now;
280 + const projection = yield* rewindUnacceptedRuns(stored);
281 let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
282 for (const run of nonterminalRuns(projection)) {
283 if (run.status === "waiting") {
@@ -380,7 +444,105 @@ export const make = Effect.gen(function* () {
444 const retryableRuns = unstartedRuns.filter((run) =>
445 retryableMessageIds.has(run.userMessageId),
446 );
383 - const retryableRunIds = new Set(retryableRuns.map((run) => run.id));
447 + // Nobody reviews a delegated, agent-sent or scheduled prompt, so it is sent again instead.
448 + const unattendedRuns = continueAfterRestart
449 + ? unstartedRuns.filter((run) =>
450 + queuedMessages.some(
451 + (message) =>
452 + message.id === run.userMessageId &&
453 + !retryableMessageIds.has(message.id) &&
454 + message.notification === undefined &&
455 + message.delegatedCompletion === undefined &&
456 + (message.scheduledTaskId !== undefined || message.creationSource === "mcp"),
457 + ),
458 + )
459 + : [];
460 + const resentRuns: Array<OrchestrationV2ThreadProjection["runs"][number]> = [];
461 + const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
462 + for (const run of unattendedRuns) {
463 + if (
464 + restartContinuationBlocked(projection, run, now, false) ||
465 + (yield* restartStopRequested(projections.getThreadRecords, projection.thread, run).pipe(
466 + Effect.mapError(
467 + (cause) =>
468 + new ProviderRuntimeRecoveryError({
469 + operation: "read-projections",
470 + threadId: projection.thread.id,
471 + cause,
472 + }),
473 + ),
474 + ))
475 + )
476 + continue;
477 + const resendCommandId = CommandId.make(`command:restart-resend:${run.id}`);
478 + const earlierResends = yield* outbox.listByCommandId(resendCommandId).pipe(
479 + Effect.mapError(
480 + (cause) =>
481 + new ProviderRuntimeRecoveryError({
482 + operation: "reconcile",
483 + threadId: projection.thread.id,
484 + cause,
485 + }),
486 + ),
487 + );
488 + // A prompt whose start keeps crashing the server would otherwise be sent forever.
489 + if (earlierResends.filter((effect) => effect.attemptCount > 0).length >= MAX_RESENDS) {
490 + yield* Effect.logWarning("Not resending a prompt that restarts cut too many times", {
491 + threadId: projection.thread.id,
492 + runId: run.id,
493 + });
494 + continue;
495 + }
496 + resentRuns.push(run);
497 + resends.push({
498 + id: `effect:restart-resend:${run.id}:${commandId}`,
499 + commandId: resendCommandId,
500 + threadId: projection.thread.id,
501 + request: { type: "provider-turn.start", runId: run.id },
502 + });
503 + }
504 + for (const run of resentRuns) {
505 + events.push({
506 + id: yield* allocateEventId(),
507 + type: "run.updated",
508 + threadId: projection.thread.id,
509 + runId: run.id,
510 + providerInstanceId: run.providerInstanceId,
511 + occurredAt: now,
512 + payload: run,
513 + });
514 + }
515 + const keptRuns = [...retryableRuns, ...resentRuns];
516 + for (const run of keptRuns) {
517 + for (const attempt of projection.attempts.filter(
518 + (candidate) => candidate.id === run.activeAttemptId && candidate.status === "running",
519 + )) {
520 + events.push({
521 + id: yield* allocateEventId(),
522 + type: "run-attempt.updated",
523 + threadId: projection.thread.id,
524 + runId: run.id,
525 + nodeId: attempt.rootNodeId,
526 + providerInstanceId: run.providerInstanceId,
527 + occurredAt: now,
528 + payload: { ...attempt, status: "pending", startedAt: null },
529 + });
530 + }
531 + for (const node of projection.nodes.filter(
532 + (candidate) => candidate.id === run.rootNodeId && candidate.status === "running",
533 + )) {
534 + events.push({
535 + id: yield* allocateEventId(),
536 + type: "node.updated",
537 + threadId: projection.thread.id,
538 + runId: run.id,
539 + nodeId: node.id,
540 + providerInstanceId: run.providerInstanceId,
541 + occurredAt: now,
542 + payload: { ...node, status: "pending", startedAt: null },
543 + });
544 + }
545 + }
546 const retryableItems =
547 retryableRuns.length === 0
548 ? []
@@ -417,7 +579,7 @@ export const make = Effect.gen(function* () {
579 },
580 });
581 }
420 - runs = runs.filter((run) => !retryableRunIds.has(run.id));
582 + runs = runs.filter((run) => !keptRuns.some((kept) => kept.id === run.id));
583 const heldRuns = [
584 ...retryableRuns,
585 ...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
@@ -932,6 +1094,7 @@ export const make = Effect.gen(function* () {
1094 closedRequests: requests.length,
1095 retiredEffects,
1096 cancelledEffectIds,
1097 + resends,
1098 };
1099 },
1100 );
@@ -951,6 +1114,7 @@ export const make = Effect.gen(function* () {
1114 let stoppedSessions = 0;
1115 let closedRequests = 0;
1116 let retiredEffects = 0;
1117 + const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
1118 const stoppedThisPass = new Map<string, RecoverySession>();
1119 const grouped = trigger === "shutdown" && threadIds.length > 1;
1120 const groupSize = grouped ? 32 : 1;
@@ -980,6 +1144,7 @@ export const make = Effect.gen(function* () {
1144 }
1145 }
1146 cancelledEffectIds.push(...result.cancelledEffectIds);
1147 + resends.push(...result.resends);
1148 terminalizedRuns += result.terminalizedRuns;
1149 stoppedSessions += result.stoppedSessions;
1150 closedRequests += result.closedRequests;
@@ -1001,6 +1166,17 @@ export const make = Effect.gen(function* () {
1166 (cause) => new ProviderRuntimeRecoveryError({ operation: "drain-outbox", cause }),
1167 ),
1168 );
1169 + // After the drain, which cancels every unsettled provider start. A crash before this write
1170 + // leaves the run unstarted, so the next recovery sends it again.
1171 + if (resends.length > 0) {
1172 + yield* eventSink
1173 + .writeWithEffects({ events: [], effects: resends })
1174 + .pipe(
1175 + Effect.mapError(
1176 + (cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
1177 + ),
1178 + );
1179 + }
1180 return {
1181 terminalizedRuns,
1182 stoppedSessions,
@@ -1036,7 +1212,10 @@ export const make = Effect.gen(function* () {
1212 .continueThreadsAfterServerUpdate
1213 )
1214 return;
1039 - const run = restartContinuationRun(projection, yield* DateTime.now);
1215 + const run = restartContinuationRun(
1216 + yield* rewindUnacceptedRuns(projection),
1217 + yield* DateTime.now,
1218 + );
1219 if (!run) return;
1220 const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
1221 yield* eventSink.writeWithEffects({
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+98 -25
@@ -149,23 +149,65 @@ it("continues a live turn whose session the adapter never marked running", () =>
149 assert.isUndefined(restartContinuationRun(withSessionStatus(status), now), status);
150 });
151
152 -it("recovers an admitted continuation after another crash before provider start", () => {
153 - const projection = makeProjection();
154 - const starting = {
155 - ...projection,
156 - runs: [
157 - {
158 - ...projection.runs[0]!,
159 - status: "starting" as const,
160 - restartContinuationOfRunId: RunId.make("run:previous-crash"),
161 - },
162 - ],
163 - providerThreads: [{ ...projection.providerThreads[0]!, status: "idle" as const }],
164 - providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
165 - providerTurns: [],
166 - };
167 - assert.equal(restartContinuationRun(starting, now)?.id, runId);
168 -});
152 +it.effect("recovers an admitted continuation after another crash before provider start", () =>
153 + Effect.gen(function* () {
154 + const projection = makeProjection();
155 + const starting = {
156 + ...projection,
157 + runs: [
158 + {
159 + ...projection.runs[0]!,
160 + status: "starting" as const,
161 + restartContinuationOfRunId: RunId.make("run:previous-crash"),
162 + },
163 + ],
164 + providerThreads: [{ ...projection.providerThreads[0]!, status: "idle" as const }],
165 + providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
166 + providerTurns: [],
167 + };
168 + assert.equal(restartContinuationRun(starting, now)?.id, runId);
169 +
170 + // Marked running, but the provider never accepted its prompt.
171 + const running = {
172 + ...projection,
173 + runs: [
174 + { ...projection.runs[0]!, restartContinuationOfRunId: RunId.make("run:previous-crash") },
175 + ],
176 + providerTurns: [],
177 + } as unknown as OrchestrationV2ThreadProjection;
178 + const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
179 + const recovery = yield* ProviderRuntimeRecovery.make.pipe(
180 + Effect.provide(
181 + Layer.mergeAll(
182 + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
183 + Layer.mock(ProjectionStore.ProjectionStoreV2)({
184 + getRecoveryThreadIds: () => Effect.succeed([threadId]),
185 + getRuntimeRecoveryProjection: () => Effect.succeed(running),
186 + getThreadRecords: () => Effect.succeed(running),
187 + }),
188 + Layer.mock(EventSink.EventSinkV2)({
189 + commitCommand: (input) =>
190 + Effect.sync(() => {
191 + commits.push(input);
192 + return { committed: true, cancelledEffectCount: 0 } as never;
193 + }),
194 + }),
195 + IdAllocator.layer,
196 + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
197 + Layer.mock(EffectOutbox.EffectOutboxV2)({
198 + get: () => Effect.succeedNone,
199 + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
200 + }),
201 + ),
202 + ),
203 + );
204 + yield* recovery.reconcile("startup");
205 + assert.deepEqual(
206 + commits.flatMap((commit) => commit.effects.map((effect) => effect.request)),
207 + [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 1 }],
208 + );
209 + }),
210 +);
211
212 it("does not continue settled root runs with restart-cancelled background work", () => {
213 const projection = makeProjection();
@@ -602,20 +644,39 @@ const devServer = {
644 work: { kind: "shell" as const, label: "vp run dev", id: "turn-item:dev-server" },
645 };
646
647 +const unacceptedFollowUp = {
648 + id: RunId.make("run:follow-up"),
649 + ordinal: 2,
650 + activeAttemptId: RunAttemptId.make("attempt:follow-up"),
651 + userMessageId: MessageId.make("message:follow-up"),
652 + status: "running",
653 + startedAt: now,
654 +};
655 +
656 it.effect.each([
606 - ["completed", scratchProjectId, backgroundSubagent, true],
607 - ["waiting", ProjectId.make("restart-project"), backgroundSubagent, true],
608 - ["completed", ProjectId.make("restart-project"), devServer, false],
609 - ["waiting", scratchProjectId, devServer, false],
657 + ["completed", scratchProjectId, backgroundSubagent, true, ""],
658 + ["waiting", ProjectId.make("restart-project"), backgroundSubagent, true, ""],
659 + ["completed", ProjectId.make("restart-project"), devServer, false, ""],
660 + ["waiting", scratchProjectId, devServer, false, ""],
661 + [
662 + "completed",
663 + ProjectId.make("restart-project"),
664 + backgroundSubagent,
665 + true,
666 + ", past a later prompt the provider never accepted",
667 + ],
668 ] as const)(
611 - "cleans up background work of a %s run in project %s and wakes it only for awaited work",
612 - ([status, projectId, background, wakes]) =>
669 + "cleans up background work of a $0 run in project $1 and wakes it only for awaited work$4",
670 + ([status, projectId, background, wakes, followUp]) =>
671 Effect.gen(function* () {
672 const base = makeProjection();
673 const projection = {
674 ...base,
675 thread: { ...base.thread, projectId },
618 - runs: [{ ...base.runs[0]!, status }],
676 + runs: [
677 + { ...base.runs[0]!, status },
678 + ...(followUp === "" ? [] : [{ ...base.runs[0]!, ...unacceptedFollowUp }]),
679 + ],
680 providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
681 turnItems: [background.turnItem],
682 subagents:
@@ -642,6 +703,16 @@ it.effect.each([
703 Layer.mock(ProjectionStore.ProjectionStoreV2)({
704 getRecoveryThreadIds: () => Effect.succeed([threadId]),
705 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
706 + getThreadRecords: () => Effect.succeed(projection),
707 + getMessagesById: () =>
708 + Effect.succeed(
709 + new Map([
710 + [
711 + unacceptedFollowUp.userMessageId,
712 + { id: unacceptedFollowUp.userMessageId, createdBy: "user" } as never,
713 + ],
714 + ]),
715 + ),
716 }),
717 Layer.mock(EventSink.EventSinkV2)({
718 writeWithEffects: (input) =>
@@ -692,7 +763,9 @@ it.effect.each([
763 assert.equal(note?.runId, runId);
764 assert.deepEqual(note?.payload.restartCancelledBackgroundWork, [background.work]);
765 if (status === "completed")
695 - assert.isFalse(events.some((event) => event.type === "run.updated"));
766 + assert.isFalse(
767 + events.some((event) => event.type === "run.updated" && event.runId === runId),
768 + );
769 }
770 }),
771 );
apps/server/src/orchestration-v2/RestartContinuation.ts
+57 -46
@@ -11,7 +11,12 @@ import {
11 } from "@t3tools/contracts";
12 import * as DateTime from "effect/DateTime";
13 import * as Effect from "effect/Effect";
14 -import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";
14 +import type {
15 + ProjectionRecordField,
16 + ProjectionRecordFilter,
17 + ProjectionRecords,
18 + ProjectionRuntimeRecoveryState,
19 +} from "./ProjectionStore.ts";
20
21 import * as ServerSettings from "../serverSettings.ts";
22 import { isNativeMaintenanceCommand } from "./CommandPolicy.ts";
@@ -36,7 +41,7 @@ const CONTINUE_PROMPT =
41 const MAX_CONTINUATION_CHAIN = 3;
42 const CONTINUATION_LOOP_WINDOW_MS = 5 * 60 * 1_000;
43
39 -function restartContinuationBlocked(
44 +export function restartContinuationBlocked(
45 projection: Pick<
46 ProjectionRuntimeRecoveryState,
47 "thread" | "runtimeRequests" | "providerTurns" | "turnItems"
@@ -84,6 +89,52 @@ function restartContinuationBlocked(
89 );
90 }
91
92 +/** A Stop from the user, or from the parent that delegated the task, that a restart cut short. */
93 +export const restartStopRequested = Effect.fn("RestartContinuation.restartStopRequested")(
94 + function* <E>(
95 + getThreadRecords: <K extends ProjectionRecordField>(
96 + threadId: ThreadId,
97 + fields: ReadonlyArray<K>,
98 + filter: ProjectionRecordFilter,
99 + ) => Effect.Effect<ProjectionRecords<K>, E>,
100 + thread: Pick<ProjectionRuntimeRecoveryState["thread"], "id" | "lineage" | "forkedFrom">,
101 + source: Pick<OrchestrationV2Run, "id" | "delegatedTaskId">,
102 + ) {
103 + const { turnItems } = yield* getThreadRecords(thread.id, ["turnItems"], {
104 + turnItemRunIds: [source.id],
105 + turnItemTypes: ["run_interrupt_request"],
106 + });
107 + if (turnItems.some((item) => item.runId === source.id && item.type === "run_interrupt_request"))
108 + return true;
109 + const { lineage, forkedFrom } = thread;
110 + const taskId =
111 + source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
112 + if (
113 + lineage.relationshipToParent !== "subagent" ||
114 + lineage.parentThreadId === null ||
115 + taskId === undefined
116 + )
117 + return false;
118 + const task = (yield* getThreadRecords(lineage.parentThreadId, ["subagents"], {
119 + subagentIds: [taskId],
120 + })).subagents.find((candidate) => candidate.id === taskId);
121 + // A settled task no longer owns later work in its thread.
122 + if (
123 + task === undefined ||
124 + task.runId === null ||
125 + task.result !== null ||
126 + !isOrchestrationV2WorkActive(task.status)
127 + )
128 + return false;
129 + return (
130 + (yield* getThreadRecords(lineage.parentThreadId, ["turnItems"], {
131 + turnItemTypes: ["run_interrupt_request"],
132 + turnItemRunIds: [task.runId],
133 + })).turnItems.length > 0
134 + );
135 + },
136 +);
137 +
138 /** App-owned delegations continue in their own threads and report back. */
139 function hasOpenAwaitedWork(
140 projection: Pick<ProjectionRuntimeRecoveryState, "runs" | "turnItems">,
@@ -254,50 +305,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
305 });
306 return;
307 }
257 - const sourceRecords = yield* threads.getThreadRecords(
258 - input.threadId,
259 - ["messages", "turnItems"],
260 - {
261 - messageIds: [source.userMessageId],
262 - turnItemRunIds: [source.id],
263 - turnItemTypes: ["run_interrupt_request"],
264 - },
265 - );
266 - if (
267 - sourceRecords.turnItems.some(
268 - (item) => item.runId === source.id && item.type === "run_interrupt_request",
269 - )
270 - )
271 - return;
272 - // A parent's Stop reaches its open delegated tasks through an effect a restart can cut short.
273 - const { lineage, forkedFrom } = projection.thread;
274 - const taskId =
275 - source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
276 - if (
277 - lineage.relationshipToParent === "subagent" &&
278 - lineage.parentThreadId !== null &&
279 - taskId !== undefined
280 - ) {
281 - const task = (yield* threads.getThreadRecords(lineage.parentThreadId, ["subagents"], {
282 - subagentIds: [taskId],
283 - })).subagents.find((candidate) => candidate.id === taskId);
284 - // A settled task no longer owns later work in its thread.
285 - const parentRunId =
286 - task !== undefined && task.result === null && isOrchestrationV2WorkActive(task.status)
287 - ? task.runId
288 - : null;
289 - if (
290 - parentRunId !== null &&
291 - (yield* threads.getThreadRecords(lineage.parentThreadId, ["turnItems"], {
292 - turnItemTypes: ["run_interrupt_request"],
293 - turnItemRunIds: [parentRunId],
294 - })).turnItems.length > 0
295 - )
296 - return;
297 - }
298 - const sourceMessage = sourceRecords.messages.find(
299 - (message) => message.id === source.userMessageId,
300 - );
308 + if (yield* restartStopRequested(threads.getThreadRecords, projection.thread, source)) return;
309 + const sourceMessage = (yield* threads.getThreadRecords(input.threadId, ["messages"], {
310 + messageIds: [source.userMessageId],
311 + })).messages.find((message) => message.id === source.userMessageId);
312 if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return;
313 const note = restartContinuationNote(
314 source,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+152
@@ -9738,6 +9738,24 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9738 occurredAt: yield* DateTime.now,
9739 payload: { ...source, status: "running", startedAt: yield* DateTime.now },
9740 },
9741 + {
9742 + id: EventId.make(`${threadId}:accepted`),
9743 + type: "provider-turn.updated",
9744 + threadId,
9745 + runId: source.id,
9746 + occurredAt: yield* DateTime.now,
9747 + payload: {
9748 + id: ProviderTurnId.make(`${threadId}:turn`),
9749 + providerThreadId: source.providerThreadId!,
9750 + nodeId: source.rootNodeId!,
9751 + runAttemptId: source.activeAttemptId,
9752 + nativeTurnRef: null,
9753 + ordinal: 1,
9754 + status: "running",
9755 + startedAt: yield* DateTime.now,
9756 + completedAt: null,
9757 + },
9758 + },
9759 ],
9760 });
9761 }
@@ -9859,6 +9877,122 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9877 }),
9878 );
9879
9880 + it.effect.each(["startup", "shutdown"] as const)(
9881 + "sends a delegated task's prompt again after a %s cut it before the provider accepted it",
9882 + (trigger) =>
9883 + Effect.gen(function* () {
9884 + const orchestrator = yield* Orchestrator.OrchestratorV2;
9885 + const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
9886 + const outbox = yield* EffectOutbox.EffectOutboxV2;
9887 + const parentThreadId = ThreadId.make(`delegate-resend-${trigger}`);
9888 + yield* orchestrator.dispatch({
9889 + type: "thread.create",
9890 + createdBy: "user",
9891 + creationSource: "web",
9892 + commandId: CommandId.make(`${parentThreadId}:create`),
9893 + threadId: parentThreadId,
9894 + projectId: ProjectId.make(`${parentThreadId}:project`),
9895 + title: "Delegating parent",
9896 + modelSelection,
9897 + runtimeMode: "full-access",
9898 + interactionMode: "default",
9899 + branch: null,
9900 + worktreePath: process.cwd(),
9901 + });
9902 + yield* orchestrator.dispatch({
9903 + type: "message.dispatch",
9904 + createdBy: "user",
9905 + creationSource: "web",
9906 + commandId: CommandId.make(`${parentThreadId}:message`),
9907 + threadId: parentThreadId,
9908 + messageId: MessageId.make(`${parentThreadId}:message`),
9909 + text: "Delegate a review",
9910 + attachments: [],
9911 + modelSelection,
9912 + dispatchMode: { type: "start_immediately" },
9913 + });
9914 + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
9915 + yield* orchestrator.dispatch({
9916 + type: "delegated_task.request",
9917 + commandId: CommandId.make(`${parentThreadId}:delegate`),
9918 + parentThreadId,
9919 + parentRunId: parentRun.id,
9920 + parentNodeId: parentRun.rootNodeId!,
9921 + task: "Review the diff",
9922 + modelSelection,
9923 + runtimeMode: "full-access",
9924 + interactionMode: "default",
9925 + completionWake: "always",
9926 + createdBy: "agent",
9927 + creationSource: "mcp",
9928 + });
9929 + const threadId = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!
9930 + .childThreadId!;
9931 + const before = yield* orchestrator.getThreadProjection(threadId);
9932 + const run = before.runs[0]!;
9933 + const attempt = before.attempts.find((row) => row.id === run.activeAttemptId)!;
9934 + const rootNode = before.nodes.find((row) => row.id === run.rootNodeId)!;
9935 + // What a provider turn start commits before the provider accepts the prompt.
9936 + const startedAt = yield* DateTime.now;
9937 + yield* (yield* EventSink.EventSinkV2).write({
9938 + events: [
9939 + {
9940 + id: EventId.make(`${threadId}:running:run`),
9941 + type: "run.updated",
9942 + threadId,
9943 + runId: run.id,
9944 + occurredAt: startedAt,
9945 + payload: { ...run, status: "running", startedAt },
9946 + },
9947 + {
9948 + id: EventId.make(`${threadId}:running:attempt`),
9949 + type: "run-attempt.updated",
9950 + threadId,
9951 + runId: run.id,
9952 + nodeId: rootNode.id,
9953 + occurredAt: startedAt,
9954 + payload: { ...attempt, status: "running", startedAt },
9955 + },
9956 + {
9957 + id: EventId.make(`${threadId}:running:node`),
9958 + type: "node.updated",
9959 + threadId,
9960 + runId: run.id,
9961 + nodeId: rootNode.id,
9962 + occurredAt: startedAt,
9963 + payload: { ...rootNode, status: "running", startedAt },
9964 + },
9965 + ],
9966 + });
9967 + const expectResent = Effect.gen(function* () {
9968 + const child = yield* orchestrator.getThreadProjection(threadId);
9969 + assert.deepEqual(
9970 + child.runs.map((row) => [row.id, row.status, row.startedAt]),
9971 + [[run.id, "starting", null]],
9972 + );
9973 + assert.deepEqual(child.messages, before.messages);
9974 + const starts = yield* outbox.listByCommandId(
9975 + CommandId.make(`command:restart-resend:${run.id}`),
9976 + );
9977 + assert.deepEqual(
9978 + starts.filter((effect) => effect.status === "pending").map((effect) => effect.request),
9979 + [{ type: "provider-turn.start", runId: run.id }],
9980 + );
9981 + });
9982 + yield* recovery.reconcile(trigger);
9983 + yield* expectResent;
9984 + // The server stops again before the worker sent the prompt.
9985 + yield* TestClock.adjust("1 second");
9986 + yield* recovery.reconcile("startup");
9987 + yield* orchestrator.recoverDelegatedTasks;
9988 + yield* expectResent;
9989 + assert.deepInclude((yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0], {
9990 + status: "running",
9991 + result: null,
9992 + });
9993 + }),
9994 + );
9995 +
9996 it.effect("starts a provider wake queued beside a held queue", () =>
9997 Effect.gen(function* () {
9998 const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -12007,6 +12141,24 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
12141 occurredAt: startedAt,
12142 payload: { ...continuation, status: "running", startedAt },
12143 },
12144 + {
12145 + id: EventId.make("recovery:continuation-accepted:resume-started"),
12146 + type: "provider-turn.updated",
12147 + threadId,
12148 + runId: continuation.id,
12149 + occurredAt: startedAt,
12150 + payload: {
12151 + id: ProviderTurnId.make("recovery:continuation-turn:resume-started"),
12152 + providerThreadId: continuation.providerThreadId!,
12153 + nodeId: continuation.rootNodeId!,
12154 + runAttemptId: continuation.activeAttemptId,
12155 + nativeTurnRef: null,
12156 + ordinal: 1,
12157 + status: "running",
12158 + startedAt,
12159 + completedAt: null,
12160 + },
12161 + },
12162 ],
12163 });
12164 }
docs/internals/server-updates.md
+8
@@ -117,6 +117,14 @@ never wake it. Its continuation prompt is the note of what will not report back.
117 unmarked shell or monitor does not wake a settled run: most are dev servers and their
118 watchdogs that nobody waits on, so that note waits for the next user message.
119
120 +A run is marked running before its provider accepts the prompt, so recovery treats a run
121 +without a provider turn as never started. A user's prompt the provider never received
122 +joins the held queue. With restart continuation on, a delegated, agent-sent, or scheduled
123 +prompt has nobody to review it, so recovery starts it again under the same run. The resend
124 +skips every thread and Stop that a continuation skips, and gives up after three tries so a
125 +start that crashes the server cannot loop. A crash after the provider received the prompt
126 +but before its turn was recorded can repeat that prompt.
127 +
128 Recovery retires effects tied to the lost process and records continuation intent
129 in the durable outbox. That intent survives another restart before provider startup.
130 Continuation effects wait for activation; a slow provider must not delay the server's