The best way to control Claude Code, Codex and more from any computer or phone
fix(server): restarts no longer drop prompts the provider never accepted (#32413)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Jolymmiles committed
Nov 15, 2026 at 08:32 UTC
34204a39e8702372edcabd129696e6acb9caca2c
8 files changed
+671
-105
apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
+10
-3
@@ -13,6 +13,7 @@ import {
13
ProviderSessionId,
14
ProviderThreadId,
15
ProviderTurnId,
16
+ RunAttemptId,
17
RunId,
18
RuntimeRequestId,
19
ThreadId,
@@ -1239,7 +1240,13 @@ it.effect("reads the run that owns a background roster, not a queued or resumed
1240
}).pipe(Effect.provide(layerTest)),
1241
);
1242
1242
-it.effect("wakes the run whose subagent lost its shell, past a wake cut before it started", () =>
1243
+it.effect.each([
1244
+ ["a wake cut before it started", { status: "starting", startedAt: null }],
1245
+ [
1246
+ "a prompt the provider never accepted",
1247
+ { status: "running", activeAttemptId: RunAttemptId.make("attempt:unaccepted") },
1248
+ ],
1249
+] as const)("wakes the run whose subagent lost its shell, past %s", ([, later]) =>
1250
Effect.gen(function* () {
1251
const projections = yield* ProjectionStore.ProjectionStoreV2;
1252
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
@@ -1269,11 +1276,11 @@ it.effect("wakes the run whose subagent lost its shell, past a wake cut before i
1276
},
1277
});
1278
const settledRun = yield* createRun(threadId, "completed", { providerThreadId });
1272
- yield* createRun(threadId, "starting", {
1279
+ yield* createRun(threadId, later.status, {
1280
ordinal: 2,
1281
providerThreadId,
1275
- startedAt: null,
1282
completedAt: null,
1283
+ ...later,
1284
});
1285
yield* projections.apply({
1286
id: EventId.make(`event:${threadId}:roster`),
apps/server/src/orchestration-v2/ProjectionStore.ts
+6
-3
@@ -5265,14 +5265,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5265
WHERE latest.thread_id = ${threadId}
5266
ORDER BY latest.ordinal DESC LIMIT 1
5267
)
5268
- -- Retain the latest run for each provider thread with lost
5269
- -- background work, including owners used before a handoff.
5268
+ -- Retain the latest ended run for each provider thread with lost
5269
+ -- background work, including owners used before a handoff. An
5270
+ -- unfinished run is read above and may prove never started.
5271
OR run.run_id IN (
5272
SELECT (
5273
SELECT ended.run_id FROM orchestration_v2_projection_runs AS ended
5274
WHERE ended.thread_id = ${threadId}
5275
AND ended.provider_thread_id = roster.provider_thread_id
5275
- AND ended.status NOT IN ('queued', 'rolled_back')
5276
+ AND ended.status NOT IN (
5277
+ 'queued', 'preparing', 'starting', 'running', 'waiting', 'rolled_back'
5278
+ )
5279
-- A run that never started (see neverStarted) owns no work.
5280
AND NOT (
5281
json_type(ended.payload_json, '$.startedAt') = 'null'
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
+156
-23
@@ -14,6 +14,7 @@ import {
14
TurnItemId,
15
type OrchestrationV2ThreadProjection,
16
} from "@t3tools/contracts";
17
+import * as DateTime from "effect/DateTime";
18
import * as Effect from "effect/Effect";
19
import * as Layer from "effect/Layer";
20
@@ -428,8 +429,17 @@ it.effect("recovery closes a secret card's form but leaves a link card open", ()
429
runtimeRequests: [],
430
providerSessions: [],
431
providerThreads: [],
431
- providerTurns: [],
432
- runs: [{ id: runId, status: "running", providerInstanceId: ProviderInstanceId.make("codex") }],
432
+ providerTurns: [
433
+ { runAttemptId: RunAttemptId.make("attempt_secret_recovery"), status: "running" },
434
+ ],
435
+ runs: [
436
+ {
437
+ id: runId,
438
+ status: "running",
439
+ activeAttemptId: RunAttemptId.make("attempt_secret_recovery"),
440
+ providerInstanceId: ProviderInstanceId.make("codex"),
441
+ },
442
+ ],
443
attempts: [],
444
nodes: [],
445
subagents: [],
@@ -508,20 +518,62 @@ it.effect.each([
518
"starting-other-request",
519
"starting-own-request",
520
"starting-scheduled",
521
+ "starting-steered",
522
+ "running",
523
+ "running-delegated",
524
+ "running-delegated-capped",
525
+ "running-delegated-snoozed",
526
+ "running-delegated-stopped",
527
+ "running-resume",
528
] as const)("recovers $0 work after restart", (scenario) => {
512
- const status = scenario.startsWith("starting") ? "starting" : scenario;
529
+ const status = scenario.startsWith("starting")
530
+ ? "starting"
531
+ : scenario.startsWith("running")
532
+ ? "running"
533
+ : scenario;
534
+ const resent = scenario === "starting-scheduled" || scenario === "running-delegated";
535
const terminalized =
536
status === "preparing" ||
537
scenario === "starting-own-request" ||
516
- scenario === "starting-scheduled";
538
+ scenario === "starting-steered" ||
539
+ scenario === "running-delegated-capped" ||
540
+ scenario === "running-delegated-snoozed" ||
541
+ scenario === "running-delegated-stopped" ||
542
+ scenario === "running-resume";
543
+ const startedAt =
544
+ status === "running" || scenario === "starting-steered"
545
+ ? DateTime.makeUnsafe("2026-11-01T19:34:40.403Z")
546
+ : null;
547
const threadId = ThreadId.make("thread_queued_restart");
548
const runId = RunId.make("run_queued_restart");
549
const attemptId = RunAttemptId.make("attempt_queued_restart");
550
+ const steeredAttemptId = RunAttemptId.make("attempt_queued_restart_steered");
551
const rootNodeId = NodeId.make("node_queued_restart");
552
let committedInput: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0] | null =
553
null;
554
+ const written: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
555
const projection = {
524
- thread: { id: threadId },
556
+ thread: {
557
+ id: threadId,
558
+ archivedAt: null,
559
+ deletedAt: null,
560
+ settledOverride: null,
561
+ snoozedUntil:
562
+ scenario === "running-delegated-snoozed"
563
+ ? DateTime.makeUnsafe("2099-07-25T09:00:00.000Z")
564
+ : null,
565
+ lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
566
+ forkedFrom: null,
567
+ ...(scenario === "running-resume"
568
+ ? {
569
+ limitRecovery: {
570
+ runId: RunId.make("run_queued_restart_limited"),
571
+ resetAt: "2026-11-01T19:34:00.000Z",
572
+ autoResume: true,
573
+ },
574
+ }
575
+ : {}),
576
+ },
577
runtimeRequests: scenario.endsWith("request")
578
? [
579
{
@@ -536,16 +588,29 @@ it.effect.each([
588
: [],
589
providerSessions: [],
590
providerThreads: [],
539
- providerTurns: [],
591
+ providerTurns:
592
+ scenario === "starting-steered"
593
+ ? [
594
+ {
595
+ id: ProviderTurnId.make("provider_turn_queued_restart_steered"),
596
+ runAttemptId: steeredAttemptId,
597
+ status: "interrupted",
598
+ },
599
+ ]
600
+ : [],
601
runs: [
602
{
603
id: runId,
604
status,
544
- startedAt: null,
605
+ startedAt,
606
+ rootNodeId,
607
activeAttemptId: attemptId,
608
userMessageId: MessageId.make("message_queued_restart"),
609
queuePosition: 1,
610
providerInstanceId: ProviderInstanceId.make("codex"),
611
+ ...(scenario === "running-resume"
612
+ ? { usageLimitContinuationOfRunId: RunId.make("run_queued_restart_limited") }
613
+ : {}),
614
...(status === "preparing"
615
? {
616
workspacePreparation: {
@@ -558,11 +623,15 @@ it.effect.each([
623
},
624
],
625
attempts: [
626
+ ...(scenario === "starting-steered"
627
+ ? [{ id: steeredAttemptId, runId, rootNodeId, status: "interrupted" }]
628
+ : []),
629
{
630
id: attemptId,
631
runId,
632
rootNodeId,
565
- status: "pending",
633
+ status: status === "running" ? "running" : "pending",
634
+ startedAt: status === "running" ? startedAt : null,
635
},
636
],
637
nodes: [
@@ -578,21 +647,37 @@ it.effect.each([
647
{
648
id: rootNodeId,
649
runId,
581
- status: "pending",
650
+ status: status === "running" ? "running" : "pending",
651
+ startedAt: status === "running" ? startedAt : null,
652
},
653
],
654
subagents: [],
655
messages:
586
- status === "starting"
656
+ status === "starting" || status === "running"
657
? [
658
{
659
id: MessageId.make("message_queued_restart"),
590
- createdBy: "user",
660
+ ...(scenario.startsWith("running-delegated")
661
+ ? { createdBy: "agent", creationSource: "mcp" }
662
+ : scenario === "running-resume"
663
+ ? { createdBy: "agent", creationSource: "server" }
664
+ : { createdBy: "user" }),
665
...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),
666
},
667
]
668
: [],
595
- turnItems: [],
669
+ turnItems:
670
+ scenario === "running-delegated-stopped"
671
+ ? [
672
+ {
673
+ id: TurnItemId.make("turn_item_queued_restart_stop"),
674
+ type: "run_interrupt_request",
675
+ runId,
676
+ nodeId: rootNodeId,
677
+ status: "completed",
678
+ },
679
+ ]
680
+ : [],
681
} as unknown as OrchestrationV2ThreadProjection;
682
const layer = ProviderRuntimeRecovery.layer.pipe(
683
Layer.provide(ServerSettings.layerTest()),
@@ -600,7 +685,12 @@ it.effect.each([
685
Layer.mergeAll(
686
Layer.mock(ProjectionStore.ProjectionStoreV2)({
687
getRecoveryThreadIds: () => Effect.succeed([threadId]),
603
- getRuntimeRecoveryProjection: () => Effect.succeed(projection),
688
+ // Like the store, recovery state carries only provider turns that have not ended.
689
+ getRuntimeRecoveryProjection: () =>
690
+ Effect.succeed({
691
+ ...projection,
692
+ providerTurns: projection.providerTurns.filter((turn) => turn.status === "running"),
693
+ }),
694
getMessagesById: (requestedThreadId, messageIds) =>
695
Effect.succeed(
696
new Map(
@@ -618,12 +708,23 @@ it.effect.each([
708
committedInput = input;
709
return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
710
},
711
+ writeWithEffects: (input) => {
712
+ written.push(...input.effects);
713
+ return Effect.succeed([]);
714
+ },
715
}),
716
IdAllocator.layer,
717
Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({
718
runRecoveryOnce: Effect.succeed(false),
719
}),
720
Layer.mock(EffectOutbox.EffectOutboxV2)({
721
+ // Three earlier restarts each cut a resend the worker had claimed.
722
+ listByCommandId: () =>
723
+ Effect.succeed(
724
+ scenario === "running-delegated-capped"
725
+ ? ([1, 1, 1].map((attemptCount) => ({ attemptCount })) as never)
726
+ : [],
727
+ ),
728
reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
729
}),
730
),
@@ -640,11 +741,15 @@ it.effect.each([
741
const runEvent = command.events.find((event) => event.type === "run.updated");
742
const attemptEvent = command.events.find((event) => event.type === "run-attempt.updated");
743
const nodeEvent = command.events.find((event) => event.type === "node.updated");
643
- assert.equal(
644
- runEvent?.type === "run.updated" ? runEvent.payload.status : null,
645
- terminalized ? "cancelled" : "queued",
646
- );
744
+ assert.deepInclude(runEvent?.type === "run.updated" ? runEvent.payload : null, {
745
+ status: terminalized ? "cancelled" : resent ? "starting" : "queued",
746
+ startedAt: scenario === "starting-steered" ? startedAt : null,
747
+ });
748
assert.isEmpty(command.effects);
749
+ assert.deepEqual(
750
+ written.map((effect) => effect.request),
751
+ resent ? [{ type: "provider-turn.start", runId }] : [],
752
+ );
753
if (terminalized) {
754
assert.equal(
755
attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,
@@ -655,10 +760,33 @@ it.effect.each([
760
scenario === "starting-own-request" ? null : "cancelled",
761
);
762
} else {
658
- assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
659
- assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queueHeld : false, true);
660
- assert.isUndefined(attemptEvent);
661
- assert.isUndefined(nodeEvent);
763
+ if (!resent) {
764
+ assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
765
+ assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queueHeld : false, true);
766
+ }
767
+ if (status === "running") {
768
+ assert.deepInclude(
769
+ attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload : null,
770
+ { status: "pending", startedAt: null },
771
+ );
772
+ assert.deepInclude(nodeEvent?.type === "node.updated" ? nodeEvent.payload : null, {
773
+ status: "pending",
774
+ startedAt: null,
775
+ });
776
+ } else {
777
+ assert.isUndefined(attemptEvent);
778
+ assert.isUndefined(nodeEvent);
779
+ }
780
+ }
781
+ // The unsent auto-resume gets a fresh request id so the limit worker sends it again.
782
+ if (scenario === "running-resume") {
783
+ const rearmed = command.events.find((event) => event.type === "thread.metadata-updated");
784
+ assert.equal(
785
+ rearmed?.type === "thread.metadata-updated"
786
+ ? rearmed.payload.limitRecovery?.requestId
787
+ : null,
788
+ command.commandId,
789
+ );
790
}
791
}).pipe(Effect.provide(layer));
792
});
@@ -707,6 +835,7 @@ it.effect(
835
{
836
id: runId,
837
status: "starting",
838
+ activeAttemptId: attemptId,
839
providerThreadId,
840
providerInstanceId: ProviderInstanceId.make("codex"),
841
},
@@ -975,7 +1104,7 @@ it.effect(
1104
pendingBackgroundTasks: [],
1105
},
1106
],
978
- providerTurns: [],
1107
+ providerTurns: [{ runAttemptId: activeAttemptId, status: "running" }],
1108
runs: [
1109
{
1110
id: settledRunId,
@@ -985,6 +1114,7 @@ it.effect(
1114
{
1115
id: activeRunId,
1116
status: "running",
1117
+ activeAttemptId,
1118
providerInstanceId: claudeInstanceId,
1119
},
1120
],
@@ -1440,7 +1570,9 @@ it.effect("leaves delegated tasks to their own child threads after process loss"
1570
pendingBackgroundTasks: [],
1571
},
1572
],
1443
- providerTurns: [],
1573
+ providerTurns: [
1574
+ { runAttemptId: RunAttemptId.make("attempt_recovery_delegation"), status: "running" },
1575
+ ],
1576
runs: [
1577
{
1578
id: settledRunId,
@@ -1453,6 +1585,7 @@ it.effect("leaves delegated tasks to their own child threads after process loss"
1585
id: runningRunId,
1586
ordinal: 2,
1587
status: "running",
1588
+ activeAttemptId: RunAttemptId.make("attempt_recovery_delegation"),
1589
providerThreadId,
1590
providerInstanceId: claudeInstanceId,
1591
},
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+184
-5
@@ -5,6 +5,7 @@ import {
5
isProviderNativeSubagentThread,
6
type OrchestrationV2DomainEvent,
7
type ProviderThreadId,
8
+ type RunAttemptId,
9
type OrchestrationV2RestartCancelledBackgroundWork,
10
type OrchestrationV2Subagent,
11
type OrchestrationV2ThreadProjection,
@@ -24,7 +25,11 @@ import * as EventSink from "./EventSink.ts";
25
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
26
import * as ProjectionStore from "./ProjectionStore.ts";
27
import * as ServerSettings from "../serverSettings.ts";
27
-import { restartContinuationRun } from "./RestartContinuation.ts";
28
+import {
29
+ restartContinuationBlocked,
30
+ restartContinuationRun,
31
+ restartStopRequested,
32
+} from "./RestartContinuation.ts";
33
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
34
import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
35
import {
@@ -54,6 +59,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
59
60
const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);
61
62
+const MAX_RESENDS = 3;
63
+
64
export interface ProviderRuntimeRecoverySummary {
65
readonly terminalizedRuns: number;
66
readonly stoppedSessions: number;
@@ -206,14 +213,71 @@ export const make = Effect.gen(function* () {
213
return [...sessions.values()];
214
};
215
216
+ /**
217
+ * A run is marked running before its provider accepts the prompt, so only a provider turn of
218
+ * one of its attempts proves the prompt arrived. Recovery treats the others as never started.
219
+ */
220
+ const rewindUnacceptedRuns = Effect.fn("ProviderRuntimeRecoveryService.rewindUnacceptedRuns")(
221
+ function* (projection: ProjectionStore.ProjectionRuntimeRecoveryState) {
222
+ const runIdByAttempt = new Map(
223
+ projection.runs.flatMap((run) =>
224
+ (run.status === "running" || (run.status === "starting" && run.startedAt !== null)) &&
225
+ run.activeAttemptId !== null
226
+ ? [[run.activeAttemptId, run.id] as const]
227
+ : [],
228
+ ),
229
+ );
230
+ if (runIdByAttempt.size === 0) return projection;
231
+ const unaccepted = new Set(runIdByAttempt.values());
232
+ const accept = (turns: ReadonlyArray<{ readonly runAttemptId: RunAttemptId | null }>) => {
233
+ for (const turn of turns) {
234
+ const runId =
235
+ turn.runAttemptId === null ? undefined : runIdByAttempt.get(turn.runAttemptId);
236
+ if (runId !== undefined) unaccepted.delete(runId);
237
+ }
238
+ };
239
+ accept(projection.providerTurns);
240
+ if (unaccepted.size === 0) return projection;
241
+ for (const attempt of projection.attempts) {
242
+ if (unaccepted.has(attempt.runId)) runIdByAttempt.set(attempt.id, attempt.runId);
243
+ }
244
+ // The recovery projection omits ended provider turns, such as one a steer replaced.
245
+ accept(
246
+ (yield* projections
247
+ .getThreadRecords(projection.thread.id, ["providerTurns"], {
248
+ providerTurnAttemptIds: [...runIdByAttempt].flatMap(([attemptId, runId]) =>
249
+ unaccepted.has(runId) ? [attemptId] : [],
250
+ ),
251
+ })
252
+ .pipe(
253
+ Effect.mapError(
254
+ (cause) =>
255
+ new ProviderRuntimeRecoveryError({
256
+ operation: "read-projections",
257
+ threadId: projection.thread.id,
258
+ cause,
259
+ }),
260
+ ),
261
+ )).providerTurns,
262
+ );
263
+ return {
264
+ ...projection,
265
+ runs: projection.runs.map((run) =>
266
+ unaccepted.has(run.id) ? { ...run, status: "starting" as const, startedAt: null } : run,
267
+ ),
268
+ };
269
+ },
270
+ );
271
+
272
const reconcileProjection = Effect.fn("ProviderRuntimeRecoveryService.reconcileProjection")(
273
function* (
211
- projection: ProjectionStore.ProjectionRuntimeRecoveryState,
274
+ stored: ProjectionStore.ProjectionRuntimeRecoveryState,
275
trigger: "startup" | "shutdown",
276
continueAfterRestart: boolean,
277
stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
278
) {
279
const now = yield* DateTime.now;
280
+ const projection = yield* rewindUnacceptedRuns(stored);
281
let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
282
for (const run of nonterminalRuns(projection)) {
283
if (run.status === "waiting") {
@@ -380,7 +444,105 @@ export const make = Effect.gen(function* () {
444
const retryableRuns = unstartedRuns.filter((run) =>
445
retryableMessageIds.has(run.userMessageId),
446
);
383
- const retryableRunIds = new Set(retryableRuns.map((run) => run.id));
447
+ // Nobody reviews a delegated, agent-sent or scheduled prompt, so it is sent again instead.
448
+ const unattendedRuns = continueAfterRestart
449
+ ? unstartedRuns.filter((run) =>
450
+ queuedMessages.some(
451
+ (message) =>
452
+ message.id === run.userMessageId &&
453
+ !retryableMessageIds.has(message.id) &&
454
+ message.notification === undefined &&
455
+ message.delegatedCompletion === undefined &&
456
+ (message.scheduledTaskId !== undefined || message.creationSource === "mcp"),
457
+ ),
458
+ )
459
+ : [];
460
+ const resentRuns: Array<OrchestrationV2ThreadProjection["runs"][number]> = [];
461
+ const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
462
+ for (const run of unattendedRuns) {
463
+ if (
464
+ restartContinuationBlocked(projection, run, now, false) ||
465
+ (yield* restartStopRequested(projections.getThreadRecords, projection.thread, run).pipe(
466
+ Effect.mapError(
467
+ (cause) =>
468
+ new ProviderRuntimeRecoveryError({
469
+ operation: "read-projections",
470
+ threadId: projection.thread.id,
471
+ cause,
472
+ }),
473
+ ),
474
+ ))
475
+ )
476
+ continue;
477
+ const resendCommandId = CommandId.make(`command:restart-resend:${run.id}`);
478
+ const earlierResends = yield* outbox.listByCommandId(resendCommandId).pipe(
479
+ Effect.mapError(
480
+ (cause) =>
481
+ new ProviderRuntimeRecoveryError({
482
+ operation: "reconcile",
483
+ threadId: projection.thread.id,
484
+ cause,
485
+ }),
486
+ ),
487
+ );
488
+ // A prompt whose start keeps crashing the server would otherwise be sent forever.
489
+ if (earlierResends.filter((effect) => effect.attemptCount > 0).length >= MAX_RESENDS) {
490
+ yield* Effect.logWarning("Not resending a prompt that restarts cut too many times", {
491
+ threadId: projection.thread.id,
492
+ runId: run.id,
493
+ });
494
+ continue;
495
+ }
496
+ resentRuns.push(run);
497
+ resends.push({
498
+ id: `effect:restart-resend:${run.id}:${commandId}`,
499
+ commandId: resendCommandId,
500
+ threadId: projection.thread.id,
501
+ request: { type: "provider-turn.start", runId: run.id },
502
+ });
503
+ }
504
+ for (const run of resentRuns) {
505
+ events.push({
506
+ id: yield* allocateEventId(),
507
+ type: "run.updated",
508
+ threadId: projection.thread.id,
509
+ runId: run.id,
510
+ providerInstanceId: run.providerInstanceId,
511
+ occurredAt: now,
512
+ payload: run,
513
+ });
514
+ }
515
+ const keptRuns = [...retryableRuns, ...resentRuns];
516
+ for (const run of keptRuns) {
517
+ for (const attempt of projection.attempts.filter(
518
+ (candidate) => candidate.id === run.activeAttemptId && candidate.status === "running",
519
+ )) {
520
+ events.push({
521
+ id: yield* allocateEventId(),
522
+ type: "run-attempt.updated",
523
+ threadId: projection.thread.id,
524
+ runId: run.id,
525
+ nodeId: attempt.rootNodeId,
526
+ providerInstanceId: run.providerInstanceId,
527
+ occurredAt: now,
528
+ payload: { ...attempt, status: "pending", startedAt: null },
529
+ });
530
+ }
531
+ for (const node of projection.nodes.filter(
532
+ (candidate) => candidate.id === run.rootNodeId && candidate.status === "running",
533
+ )) {
534
+ events.push({
535
+ id: yield* allocateEventId(),
536
+ type: "node.updated",
537
+ threadId: projection.thread.id,
538
+ runId: run.id,
539
+ nodeId: node.id,
540
+ providerInstanceId: run.providerInstanceId,
541
+ occurredAt: now,
542
+ payload: { ...node, status: "pending", startedAt: null },
543
+ });
544
+ }
545
+ }
546
const retryableItems =
547
retryableRuns.length === 0
548
? []
@@ -417,7 +579,7 @@ export const make = Effect.gen(function* () {
579
},
580
});
581
}
420
- runs = runs.filter((run) => !retryableRunIds.has(run.id));
582
+ runs = runs.filter((run) => !keptRuns.some((kept) => kept.id === run.id));
583
const heldRuns = [
584
...retryableRuns,
585
...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
@@ -932,6 +1094,7 @@ export const make = Effect.gen(function* () {
1094
closedRequests: requests.length,
1095
retiredEffects,
1096
cancelledEffectIds,
1097
+ resends,
1098
};
1099
},
1100
);
@@ -951,6 +1114,7 @@ export const make = Effect.gen(function* () {
1114
let stoppedSessions = 0;
1115
let closedRequests = 0;
1116
let retiredEffects = 0;
1117
+ const resends: Array<EffectOutbox.PendingOrchestrationEffectV2> = [];
1118
const stoppedThisPass = new Map<string, RecoverySession>();
1119
const grouped = trigger === "shutdown" && threadIds.length > 1;
1120
const groupSize = grouped ? 32 : 1;
@@ -980,6 +1144,7 @@ export const make = Effect.gen(function* () {
1144
}
1145
}
1146
cancelledEffectIds.push(...result.cancelledEffectIds);
1147
+ resends.push(...result.resends);
1148
terminalizedRuns += result.terminalizedRuns;
1149
stoppedSessions += result.stoppedSessions;
1150
closedRequests += result.closedRequests;
@@ -1001,6 +1166,17 @@ export const make = Effect.gen(function* () {
1166
(cause) => new ProviderRuntimeRecoveryError({ operation: "drain-outbox", cause }),
1167
),
1168
);
1169
+ // After the drain, which cancels every unsettled provider start. A crash before this write
1170
+ // leaves the run unstarted, so the next recovery sends it again.
1171
+ if (resends.length > 0) {
1172
+ yield* eventSink
1173
+ .writeWithEffects({ events: [], effects: resends })
1174
+ .pipe(
1175
+ Effect.mapError(
1176
+ (cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
1177
+ ),
1178
+ );
1179
+ }
1180
return {
1181
terminalizedRuns,
1182
stoppedSessions,
@@ -1036,7 +1212,10 @@ export const make = Effect.gen(function* () {
1212
.continueThreadsAfterServerUpdate
1213
)
1214
return;
1039
- const run = restartContinuationRun(projection, yield* DateTime.now);
1215
+ const run = restartContinuationRun(
1216
+ yield* rewindUnacceptedRuns(projection),
1217
+ yield* DateTime.now,
1218
+ );
1219
if (!run) return;
1220
const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
1221
yield* eventSink.writeWithEffects({
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+98
-25
@@ -149,23 +149,65 @@ it("continues a live turn whose session the adapter never marked running", () =>
149
assert.isUndefined(restartContinuationRun(withSessionStatus(status), now), status);
150
});
151
152
-it("recovers an admitted continuation after another crash before provider start", () => {
153
- const projection = makeProjection();
154
- const starting = {
155
- ...projection,
156
- runs: [
157
- {
158
- ...projection.runs[0]!,
159
- status: "starting" as const,
160
- restartContinuationOfRunId: RunId.make("run:previous-crash"),
161
- },
162
- ],
163
- providerThreads: [{ ...projection.providerThreads[0]!, status: "idle" as const }],
164
- providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
165
- providerTurns: [],
166
- };
167
- assert.equal(restartContinuationRun(starting, now)?.id, runId);
168
-});
152
+it.effect("recovers an admitted continuation after another crash before provider start", () =>
153
+ Effect.gen(function* () {
154
+ const projection = makeProjection();
155
+ const starting = {
156
+ ...projection,
157
+ runs: [
158
+ {
159
+ ...projection.runs[0]!,
160
+ status: "starting" as const,
161
+ restartContinuationOfRunId: RunId.make("run:previous-crash"),
162
+ },
163
+ ],
164
+ providerThreads: [{ ...projection.providerThreads[0]!, status: "idle" as const }],
165
+ providerSessions: [{ ...projection.providerSessions[0]!, status: "stopped" as const }],
166
+ providerTurns: [],
167
+ };
168
+ assert.equal(restartContinuationRun(starting, now)?.id, runId);
169
+
170
+ // Marked running, but the provider never accepted its prompt.
171
+ const running = {
172
+ ...projection,
173
+ runs: [
174
+ { ...projection.runs[0]!, restartContinuationOfRunId: RunId.make("run:previous-crash") },
175
+ ],
176
+ providerTurns: [],
177
+ } as unknown as OrchestrationV2ThreadProjection;
178
+ const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
179
+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(
180
+ Effect.provide(
181
+ Layer.mergeAll(
182
+ ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
183
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
184
+ getRecoveryThreadIds: () => Effect.succeed([threadId]),
185
+ getRuntimeRecoveryProjection: () => Effect.succeed(running),
186
+ getThreadRecords: () => Effect.succeed(running),
187
+ }),
188
+ Layer.mock(EventSink.EventSinkV2)({
189
+ commitCommand: (input) =>
190
+ Effect.sync(() => {
191
+ commits.push(input);
192
+ return { committed: true, cancelledEffectCount: 0 } as never;
193
+ }),
194
+ }),
195
+ IdAllocator.layer,
196
+ Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
197
+ Layer.mock(EffectOutbox.EffectOutboxV2)({
198
+ get: () => Effect.succeedNone,
199
+ reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
200
+ }),
201
+ ),
202
+ ),
203
+ );
204
+ yield* recovery.reconcile("startup");
205
+ assert.deepEqual(
206
+ commits.flatMap((commit) => commit.effects.map((effect) => effect.request)),
207
+ [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 1 }],
208
+ );
209
+ }),
210
+);
211
212
it("does not continue settled root runs with restart-cancelled background work", () => {
213
const projection = makeProjection();
@@ -602,20 +644,39 @@ const devServer = {
644
work: { kind: "shell" as const, label: "vp run dev", id: "turn-item:dev-server" },
645
};
646
647
+const unacceptedFollowUp = {
648
+ id: RunId.make("run:follow-up"),
649
+ ordinal: 2,
650
+ activeAttemptId: RunAttemptId.make("attempt:follow-up"),
651
+ userMessageId: MessageId.make("message:follow-up"),
652
+ status: "running",
653
+ startedAt: now,
654
+};
655
+
656
it.effect.each([
606
- ["completed", scratchProjectId, backgroundSubagent, true],
607
- ["waiting", ProjectId.make("restart-project"), backgroundSubagent, true],
608
- ["completed", ProjectId.make("restart-project"), devServer, false],
609
- ["waiting", scratchProjectId, devServer, false],
657
+ ["completed", scratchProjectId, backgroundSubagent, true, ""],
658
+ ["waiting", ProjectId.make("restart-project"), backgroundSubagent, true, ""],
659
+ ["completed", ProjectId.make("restart-project"), devServer, false, ""],
660
+ ["waiting", scratchProjectId, devServer, false, ""],
661
+ [
662
+ "completed",
663
+ ProjectId.make("restart-project"),
664
+ backgroundSubagent,
665
+ true,
666
+ ", past a later prompt the provider never accepted",
667
+ ],
668
] as const)(
611
- "cleans up background work of a %s run in project %s and wakes it only for awaited work",
612
- ([status, projectId, background, wakes]) =>
669
+ "cleans up background work of a $0 run in project $1 and wakes it only for awaited work$4",
670
+ ([status, projectId, background, wakes, followUp]) =>
671
Effect.gen(function* () {
672
const base = makeProjection();
673
const projection = {
674
...base,
675
thread: { ...base.thread, projectId },
618
- runs: [{ ...base.runs[0]!, status }],
676
+ runs: [
677
+ { ...base.runs[0]!, status },
678
+ ...(followUp === "" ? [] : [{ ...base.runs[0]!, ...unacceptedFollowUp }]),
679
+ ],
680
providerTurns: [{ ...base.providerTurns[0]!, status: "completed" }],
681
turnItems: [background.turnItem],
682
subagents:
@@ -642,6 +703,16 @@ it.effect.each([
703
Layer.mock(ProjectionStore.ProjectionStoreV2)({
704
getRecoveryThreadIds: () => Effect.succeed([threadId]),
705
getRuntimeRecoveryProjection: () => Effect.succeed(projection),
706
+ getThreadRecords: () => Effect.succeed(projection),
707
+ getMessagesById: () =>
708
+ Effect.succeed(
709
+ new Map([
710
+ [
711
+ unacceptedFollowUp.userMessageId,
712
+ { id: unacceptedFollowUp.userMessageId, createdBy: "user" } as never,
713
+ ],
714
+ ]),
715
+ ),
716
}),
717
Layer.mock(EventSink.EventSinkV2)({
718
writeWithEffects: (input) =>
@@ -692,7 +763,9 @@ it.effect.each([
763
assert.equal(note?.runId, runId);
764
assert.deepEqual(note?.payload.restartCancelledBackgroundWork, [background.work]);
765
if (status === "completed")
695
- assert.isFalse(events.some((event) => event.type === "run.updated"));
766
+ assert.isFalse(
767
+ events.some((event) => event.type === "run.updated" && event.runId === runId),
768
+ );
769
}
770
}),
771
);
apps/server/src/orchestration-v2/RestartContinuation.ts
+57
-46
@@ -11,7 +11,12 @@ import {
11
} from "@t3tools/contracts";
12
import * as DateTime from "effect/DateTime";
13
import * as Effect from "effect/Effect";
14
-import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";
14
+import type {
15
+ ProjectionRecordField,
16
+ ProjectionRecordFilter,
17
+ ProjectionRecords,
18
+ ProjectionRuntimeRecoveryState,
19
+} from "./ProjectionStore.ts";
20
21
import * as ServerSettings from "../serverSettings.ts";
22
import { isNativeMaintenanceCommand } from "./CommandPolicy.ts";
@@ -36,7 +41,7 @@ const CONTINUE_PROMPT =
41
const MAX_CONTINUATION_CHAIN = 3;
42
const CONTINUATION_LOOP_WINDOW_MS = 5 * 60 * 1_000;
43
39
-function restartContinuationBlocked(
44
+export function restartContinuationBlocked(
45
projection: Pick<
46
ProjectionRuntimeRecoveryState,
47
"thread" | "runtimeRequests" | "providerTurns" | "turnItems"
@@ -84,6 +89,52 @@ function restartContinuationBlocked(
89
);
90
}
91
92
+/** A Stop from the user, or from the parent that delegated the task, that a restart cut short. */
93
+export const restartStopRequested = Effect.fn("RestartContinuation.restartStopRequested")(
94
+ function* <E>(
95
+ getThreadRecords: <K extends ProjectionRecordField>(
96
+ threadId: ThreadId,
97
+ fields: ReadonlyArray<K>,
98
+ filter: ProjectionRecordFilter,
99
+ ) => Effect.Effect<ProjectionRecords<K>, E>,
100
+ thread: Pick<ProjectionRuntimeRecoveryState["thread"], "id" | "lineage" | "forkedFrom">,
101
+ source: Pick<OrchestrationV2Run, "id" | "delegatedTaskId">,
102
+ ) {
103
+ const { turnItems } = yield* getThreadRecords(thread.id, ["turnItems"], {
104
+ turnItemRunIds: [source.id],
105
+ turnItemTypes: ["run_interrupt_request"],
106
+ });
107
+ if (turnItems.some((item) => item.runId === source.id && item.type === "run_interrupt_request"))
108
+ return true;
109
+ const { lineage, forkedFrom } = thread;
110
+ const taskId =
111
+ source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
112
+ if (
113
+ lineage.relationshipToParent !== "subagent" ||
114
+ lineage.parentThreadId === null ||
115
+ taskId === undefined
116
+ )
117
+ return false;
118
+ const task = (yield* getThreadRecords(lineage.parentThreadId, ["subagents"], {
119
+ subagentIds: [taskId],
120
+ })).subagents.find((candidate) => candidate.id === taskId);
121
+ // A settled task no longer owns later work in its thread.
122
+ if (
123
+ task === undefined ||
124
+ task.runId === null ||
125
+ task.result !== null ||
126
+ !isOrchestrationV2WorkActive(task.status)
127
+ )
128
+ return false;
129
+ return (
130
+ (yield* getThreadRecords(lineage.parentThreadId, ["turnItems"], {
131
+ turnItemTypes: ["run_interrupt_request"],
132
+ turnItemRunIds: [task.runId],
133
+ })).turnItems.length > 0
134
+ );
135
+ },
136
+);
137
+
138
/** App-owned delegations continue in their own threads and report back. */
139
function hasOpenAwaitedWork(
140
projection: Pick<ProjectionRuntimeRecoveryState, "runs" | "turnItems">,
@@ -254,50 +305,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
305
});
306
return;
307
}
257
- const sourceRecords = yield* threads.getThreadRecords(
258
- input.threadId,
259
- ["messages", "turnItems"],
260
- {
261
- messageIds: [source.userMessageId],
262
- turnItemRunIds: [source.id],
263
- turnItemTypes: ["run_interrupt_request"],
264
- },
265
- );
266
- if (
267
- sourceRecords.turnItems.some(
268
- (item) => item.runId === source.id && item.type === "run_interrupt_request",
269
- )
270
- )
271
- return;
272
- // A parent's Stop reaches its open delegated tasks through an effect a restart can cut short.
273
- const { lineage, forkedFrom } = projection.thread;
274
- const taskId =
275
- source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
276
- if (
277
- lineage.relationshipToParent === "subagent" &&
278
- lineage.parentThreadId !== null &&
279
- taskId !== undefined
280
- ) {
281
- const task = (yield* threads.getThreadRecords(lineage.parentThreadId, ["subagents"], {
282
- subagentIds: [taskId],
283
- })).subagents.find((candidate) => candidate.id === taskId);
284
- // A settled task no longer owns later work in its thread.
285
- const parentRunId =
286
- task !== undefined && task.result === null && isOrchestrationV2WorkActive(task.status)
287
- ? task.runId
288
- : null;
289
- if (
290
- parentRunId !== null &&
291
- (yield* threads.getThreadRecords(lineage.parentThreadId, ["turnItems"], {
292
- turnItemTypes: ["run_interrupt_request"],
293
- turnItemRunIds: [parentRunId],
294
- })).turnItems.length > 0
295
- )
296
- return;
297
- }
298
- const sourceMessage = sourceRecords.messages.find(
299
- (message) => message.id === source.userMessageId,
300
- );
308
+ if (yield* restartStopRequested(threads.getThreadRecords, projection.thread, source)) return;
309
+ const sourceMessage = (yield* threads.getThreadRecords(input.threadId, ["messages"], {
310
+ messageIds: [source.userMessageId],
311
+ })).messages.find((message) => message.id === source.userMessageId);
312
if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return;
313
const note = restartContinuationNote(
314
source,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+152
@@ -9738,6 +9738,24 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9738
occurredAt: yield* DateTime.now,
9739
payload: { ...source, status: "running", startedAt: yield* DateTime.now },
9740
},
9741
+ {
9742
+ id: EventId.make(`${threadId}:accepted`),
9743
+ type: "provider-turn.updated",
9744
+ threadId,
9745
+ runId: source.id,
9746
+ occurredAt: yield* DateTime.now,
9747
+ payload: {
9748
+ id: ProviderTurnId.make(`${threadId}:turn`),
9749
+ providerThreadId: source.providerThreadId!,
9750
+ nodeId: source.rootNodeId!,
9751
+ runAttemptId: source.activeAttemptId,
9752
+ nativeTurnRef: null,
9753
+ ordinal: 1,
9754
+ status: "running",
9755
+ startedAt: yield* DateTime.now,
9756
+ completedAt: null,
9757
+ },
9758
+ },
9759
],
9760
});
9761
}
@@ -9859,6 +9877,122 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9877
}),
9878
);
9879
9880
+ it.effect.each(["startup", "shutdown"] as const)(
9881
+ "sends a delegated task's prompt again after a %s cut it before the provider accepted it",
9882
+ (trigger) =>
9883
+ Effect.gen(function* () {
9884
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
9885
+ const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
9886
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
9887
+ const parentThreadId = ThreadId.make(`delegate-resend-${trigger}`);
9888
+ yield* orchestrator.dispatch({
9889
+ type: "thread.create",
9890
+ createdBy: "user",
9891
+ creationSource: "web",
9892
+ commandId: CommandId.make(`${parentThreadId}:create`),
9893
+ threadId: parentThreadId,
9894
+ projectId: ProjectId.make(`${parentThreadId}:project`),
9895
+ title: "Delegating parent",
9896
+ modelSelection,
9897
+ runtimeMode: "full-access",
9898
+ interactionMode: "default",
9899
+ branch: null,
9900
+ worktreePath: process.cwd(),
9901
+ });
9902
+ yield* orchestrator.dispatch({
9903
+ type: "message.dispatch",
9904
+ createdBy: "user",
9905
+ creationSource: "web",
9906
+ commandId: CommandId.make(`${parentThreadId}:message`),
9907
+ threadId: parentThreadId,
9908
+ messageId: MessageId.make(`${parentThreadId}:message`),
9909
+ text: "Delegate a review",
9910
+ attachments: [],
9911
+ modelSelection,
9912
+ dispatchMode: { type: "start_immediately" },
9913
+ });
9914
+ const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
9915
+ yield* orchestrator.dispatch({
9916
+ type: "delegated_task.request",
9917
+ commandId: CommandId.make(`${parentThreadId}:delegate`),
9918
+ parentThreadId,
9919
+ parentRunId: parentRun.id,
9920
+ parentNodeId: parentRun.rootNodeId!,
9921
+ task: "Review the diff",
9922
+ modelSelection,
9923
+ runtimeMode: "full-access",
9924
+ interactionMode: "default",
9925
+ completionWake: "always",
9926
+ createdBy: "agent",
9927
+ creationSource: "mcp",
9928
+ });
9929
+ const threadId = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!
9930
+ .childThreadId!;
9931
+ const before = yield* orchestrator.getThreadProjection(threadId);
9932
+ const run = before.runs[0]!;
9933
+ const attempt = before.attempts.find((row) => row.id === run.activeAttemptId)!;
9934
+ const rootNode = before.nodes.find((row) => row.id === run.rootNodeId)!;
9935
+ // What a provider turn start commits before the provider accepts the prompt.
9936
+ const startedAt = yield* DateTime.now;
9937
+ yield* (yield* EventSink.EventSinkV2).write({
9938
+ events: [
9939
+ {
9940
+ id: EventId.make(`${threadId}:running:run`),
9941
+ type: "run.updated",
9942
+ threadId,
9943
+ runId: run.id,
9944
+ occurredAt: startedAt,
9945
+ payload: { ...run, status: "running", startedAt },
9946
+ },
9947
+ {
9948
+ id: EventId.make(`${threadId}:running:attempt`),
9949
+ type: "run-attempt.updated",
9950
+ threadId,
9951
+ runId: run.id,
9952
+ nodeId: rootNode.id,
9953
+ occurredAt: startedAt,
9954
+ payload: { ...attempt, status: "running", startedAt },
9955
+ },
9956
+ {
9957
+ id: EventId.make(`${threadId}:running:node`),
9958
+ type: "node.updated",
9959
+ threadId,
9960
+ runId: run.id,
9961
+ nodeId: rootNode.id,
9962
+ occurredAt: startedAt,
9963
+ payload: { ...rootNode, status: "running", startedAt },
9964
+ },
9965
+ ],
9966
+ });
9967
+ const expectResent = Effect.gen(function* () {
9968
+ const child = yield* orchestrator.getThreadProjection(threadId);
9969
+ assert.deepEqual(
9970
+ child.runs.map((row) => [row.id, row.status, row.startedAt]),
9971
+ [[run.id, "starting", null]],
9972
+ );
9973
+ assert.deepEqual(child.messages, before.messages);
9974
+ const starts = yield* outbox.listByCommandId(
9975
+ CommandId.make(`command:restart-resend:${run.id}`),
9976
+ );
9977
+ assert.deepEqual(
9978
+ starts.filter((effect) => effect.status === "pending").map((effect) => effect.request),
9979
+ [{ type: "provider-turn.start", runId: run.id }],
9980
+ );
9981
+ });
9982
+ yield* recovery.reconcile(trigger);
9983
+ yield* expectResent;
9984
+ // The server stops again before the worker sent the prompt.
9985
+ yield* TestClock.adjust("1 second");
9986
+ yield* recovery.reconcile("startup");
9987
+ yield* orchestrator.recoverDelegatedTasks;
9988
+ yield* expectResent;
9989
+ assert.deepInclude((yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0], {
9990
+ status: "running",
9991
+ result: null,
9992
+ });
9993
+ }),
9994
+ );
9995
+
9996
it.effect("starts a provider wake queued beside a held queue", () =>
9997
Effect.gen(function* () {
9998
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -12007,6 +12141,24 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
12141
occurredAt: startedAt,
12142
payload: { ...continuation, status: "running", startedAt },
12143
},
12144
+ {
12145
+ id: EventId.make("recovery:continuation-accepted:resume-started"),
12146
+ type: "provider-turn.updated",
12147
+ threadId,
12148
+ runId: continuation.id,
12149
+ occurredAt: startedAt,
12150
+ payload: {
12151
+ id: ProviderTurnId.make("recovery:continuation-turn:resume-started"),
12152
+ providerThreadId: continuation.providerThreadId!,
12153
+ nodeId: continuation.rootNodeId!,
12154
+ runAttemptId: continuation.activeAttemptId,
12155
+ nativeTurnRef: null,
12156
+ ordinal: 1,
12157
+ status: "running",
12158
+ startedAt,
12159
+ completedAt: null,
12160
+ },
12161
+ },
12162
],
12163
});
12164
}
docs/internals/server-updates.md
+8
@@ -117,6 +117,14 @@ never wake it. Its continuation prompt is the note of what will not report back.
117
unmarked shell or monitor does not wake a settled run: most are dev servers and their
118
watchdogs that nobody waits on, so that note waits for the next user message.
119
120
+A run is marked running before its provider accepts the prompt, so recovery treats a run
121
+without a provider turn as never started. A user's prompt the provider never received
122
+joins the held queue. With restart continuation on, a delegated, agent-sent, or scheduled
123
+prompt has nobody to review it, so recovery starts it again under the same run. The resend
124
+skips every thread and Stop that a continuation skips, and gives up after three tries so a
125
+start that crashes the server cannot loop. A crash after the provider received the prompt
126
+but before its turn was recorded can repeat that prompt.
127
+
128
Recovery retires effects tied to the lost process and records continuation intent
129
in the durable outbox. That intent survives another restart before provider startup.
130
Continuation effects wait for activation; a slow provider must not delay the server's