The best way to control Claude Code, Codex and more from any computer or phone
fix(server): retry unstarted runs after recovery (#30560)
Yash Singh committed
Nov 3, 2026 at 20:54 UTC
737afe24bd20d1390f36fa6f5d4e7974dffead3c
5 files changed
+233
-39
apps/server/src/orchestration-v2/Orchestrator.ts
+2
-1
@@ -2376,7 +2376,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2376
},
2377
})),
2378
{
2379
- id: `effect:${commandId}:provider-turn.start:${queuedRun.id}`,
2379
+ // A restart can return this run to the queue after its previous start effect was cancelled.
2380
+ id: `effect:${commandId}:provider-turn.start:${queuedRun.id}:${yield* idAllocator.allocate.event({ threadId, commandId })}`,
2381
commandId,
2382
threadId,
2383
request: { type: "provider-turn.start", runId: queuedRun.id },
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
+51
-7
@@ -501,7 +501,19 @@ it.effect("recovery closes a secret card's form but leaves a link card open", ()
501
}).pipe(Effect.provide(layer));
502
});
503
504
-it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart", (status) => {
504
+it.effect.each([
505
+ "queued",
506
+ "preparing",
507
+ "starting",
508
+ "starting-other-request",
509
+ "starting-own-request",
510
+ "starting-scheduled",
511
+] as const)("recovers $0 work after restart", (scenario) => {
512
+ const status = scenario.startsWith("starting") ? "starting" : scenario;
513
+ const terminalized =
514
+ status === "preparing" ||
515
+ scenario === "starting-own-request" ||
516
+ scenario === "starting-scheduled";
517
const threadId = ThreadId.make("thread_queued_restart");
518
const runId = RunId.make("run_queued_restart");
519
const attemptId = RunAttemptId.make("attempt_queued_restart");
@@ -510,7 +522,18 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
522
null;
523
const projection = {
524
thread: { id: threadId },
513
- runtimeRequests: [],
525
+ runtimeRequests: scenario.endsWith("request")
526
+ ? [
527
+ {
528
+ id: RuntimeRequestId.make("pending-recovery-request"),
529
+ nodeId:
530
+ scenario === "starting-own-request" ? rootNodeId : NodeId.make("other-run-request"),
531
+ kind: "user_input",
532
+ status: "pending",
533
+ responseCapability: { type: "message" },
534
+ },
535
+ ]
536
+ : [],
537
providerSessions: [],
538
providerThreads: [],
539
providerTurns: [],
@@ -518,6 +541,9 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
541
{
542
id: runId,
543
status,
544
+ startedAt: null,
545
+ activeAttemptId: attemptId,
546
+ userMessageId: MessageId.make("message_queued_restart"),
547
queuePosition: 1,
548
providerInstanceId: ProviderInstanceId.make("codex"),
549
...(status === "preparing"
@@ -540,6 +566,15 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
566
},
567
],
568
nodes: [
569
+ ...(scenario === "starting-other-request"
570
+ ? [
571
+ {
572
+ id: NodeId.make("other-run-request"),
573
+ runId: RunId.make("earlier-run"),
574
+ status: "waiting",
575
+ },
576
+ ]
577
+ : []),
578
{
579
id: rootNodeId,
580
runId,
@@ -547,7 +582,16 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
582
},
583
],
584
subagents: [],
550
- messages: [],
585
+ messages:
586
+ status === "starting"
587
+ ? [
588
+ {
589
+ id: MessageId.make("message_queued_restart"),
590
+ createdBy: "user",
591
+ ...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),
592
+ },
593
+ ]
594
+ : [],
595
turnItems: [],
596
} as unknown as OrchestrationV2ThreadProjection;
597
const layer = ProviderRuntimeRecovery.layer.pipe(
@@ -579,7 +623,7 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
623
return Effect.gen(function* () {
624
const summary =
625
yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
582
- assert.equal(summary.terminalizedRuns, status === "preparing" ? 1 : 0);
626
+ assert.equal(summary.terminalizedRuns, terminalized ? 1 : 0);
627
const command = committedInput;
628
assert.isNotNull(command);
629
if (command === null) return;
@@ -588,17 +632,17 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
632
const nodeEvent = command.events.find((event) => event.type === "node.updated");
633
assert.equal(
634
runEvent?.type === "run.updated" ? runEvent.payload.status : null,
591
- status === "preparing" ? "cancelled" : "queued",
635
+ terminalized ? "cancelled" : "queued",
636
);
637
assert.isEmpty(command.effects);
594
- if (status === "preparing") {
638
+ if (terminalized) {
639
assert.equal(
640
attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,
641
"cancelled",
642
);
643
assert.equal(
644
nodeEvent?.type === "node.updated" ? nodeEvent.payload.status : null,
601
- "cancelled",
645
+ scenario === "starting-own-request" ? null : "cancelled",
646
);
647
} else {
648
assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+81
-6
@@ -25,6 +25,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
25
import * as ServerSettings from "../serverSettings.ts";
26
import { restartContinuationRun } from "./RestartContinuation.ts";
27
import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
28
+import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
29
import {
30
endOrphanedNativeSubagent,
31
endRunlessRootTurns,
@@ -210,7 +211,7 @@ export const make = Effect.gen(function* () {
211
stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
212
) {
213
const now = yield* DateTime.now;
213
- const runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
214
+ let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
215
for (const run of nonterminalRuns(projection)) {
216
if (run.status === "waiting") {
217
const checkpointEffects = yield* outbox
@@ -325,13 +326,25 @@ export const make = Effect.gen(function* () {
326
cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);
327
}
328
};
329
+ const unstartedRuns = runs.filter(
330
+ (run) =>
331
+ run.status === "starting" &&
332
+ neverStarted(run) &&
333
+ !projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) &&
334
+ !projection.runtimeRequests.some(
335
+ (request) =>
336
+ request.status === "pending" &&
337
+ projection.nodes.some((node) => node.id === request.nodeId && node.runId === run.id),
338
+ ),
339
+ );
340
const queuedRuns = projection.runs.filter((run) => run.status === "queued");
341
+ const pendingRuns = [...unstartedRuns, ...queuedRuns];
342
const queuedMessages =
330
- queuedRuns.length === 0
343
+ pendingRuns.length === 0
344
? []
345
: (yield* projections
346
.getThreadRecords(projection.thread.id, ["messages"], {
334
- messageIds: queuedRuns.map((run) => run.userMessageId),
347
+ messageIds: pendingRuns.map((run) => run.userMessageId),
348
})
349
.pipe(
350
Effect.mapError(
@@ -346,12 +359,69 @@ export const make = Effect.gen(function* () {
359
const bufferedMessageIds = new Set(
360
queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
361
);
349
- for (const run of queuedRuns) {
362
+ const retryableMessageIds = new Set(
363
+ queuedMessages
364
+ .filter(
365
+ (message) =>
366
+ message.createdBy === "user" &&
367
+ message.scheduledTaskId === undefined &&
368
+ message.notification === undefined &&
369
+ message.delegatedCompletion === undefined,
370
+ )
371
+ .map((message) => message.id),
372
+ );
373
+ // A saved user message the provider never accepted stays available through the held queue.
374
+ const retryableRuns = unstartedRuns.filter((run) =>
375
+ retryableMessageIds.has(run.userMessageId),
376
+ );
377
+ const retryableRunIds = new Set(retryableRuns.map((run) => run.id));
378
+ const retryableItems =
379
+ retryableRuns.length === 0
380
+ ? []
381
+ : (yield* projections
382
+ .getThreadRecords(projection.thread.id, ["turnItems"], {
383
+ turnItemTypes: ["user_message"],
384
+ turnItemRunIds: retryableRuns.map((run) => run.id),
385
+ })
386
+ .pipe(
387
+ Effect.mapError(
388
+ (cause) =>
389
+ new ProviderRuntimeRecoveryError({
390
+ operation: "read-projections",
391
+ threadId: projection.thread.id,
392
+ cause,
393
+ }),
394
+ ),
395
+ )).turnItems;
396
+ for (const item of retryableItems) {
397
+ if (item.type !== "user_message" || !retryableMessageIds.has(item.messageId)) continue;
398
+ events.push({
399
+ id: yield* allocateEventId(),
400
+ type: "turn-item.updated",
401
+ threadId: projection.thread.id,
402
+ ...(item.runId === null ? {} : { runId: item.runId }),
403
+ occurredAt: now,
404
+ payload: {
405
+ ...item,
406
+ inputIntent: "queued_turn",
407
+ status: "pending",
408
+ startedAt: null,
409
+ completedAt: null,
410
+ updatedAt: now,
411
+ },
412
+ });
413
+ }
414
+ runs = runs.filter((run) => !retryableRunIds.has(run.id));
415
+ const heldRuns = [
416
+ ...retryableRuns,
417
+ ...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
418
+ ];
419
+ for (const [index, run] of heldRuns.entries()) {
420
if (bufferedMessageIds.has(run.userMessageId)) {
421
runs.push(run);
422
continue;
423
}
354
- if (run.queueHeld === true) continue;
424
+ if (run.queueHeld === true && retryableRuns.length === 0) continue;
425
events.push({
426
id: yield* allocateEventId(),
427
type: "run.updated",
@@ -359,7 +429,12 @@ export const make = Effect.gen(function* () {
429
runId: run.id,
430
providerInstanceId: run.providerInstanceId,
431
occurredAt: now,
362
- payload: { ...run, queueHeld: true },
432
+ payload: {
433
+ ...run,
434
+ status: "queued",
435
+ queueHeld: true,
436
+ ...(retryableRuns.length === 0 ? {} : { queuePosition: index + 1 }),
437
+ },
438
});
439
}
440
for (const question of questions) {
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+98
-24
@@ -6310,13 +6310,18 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6310
}),
6311
);
6312
6313
- it.effect.each(["startup", "shutdown"] as const)(
6314
- "preserves and holds queued messages across %s until explicitly resumed",
6315
- (trigger) =>
6313
+ it.effect.each([
6314
+ { trigger: "startup", started: true },
6315
+ { trigger: "shutdown", started: true },
6316
+ { trigger: "startup", started: false },
6317
+ { trigger: "shutdown", started: false },
6318
+ ] as const)(
6319
+ "preserves and holds queued messages across $trigger with started=$started until explicitly resumed",
6320
+ ({ trigger, started }) =>
6321
Effect.gen(function* () {
6322
const orchestrator = yield* Orchestrator.OrchestratorV2;
6323
const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
6319
- const threadId = ThreadId.make(`queue-hold-${trigger}`);
6324
+ const threadId = ThreadId.make(`queue-hold-${trigger}-${started}`);
6325
yield* orchestrator.dispatch({
6326
type: "thread.create",
6327
createdBy: "user",
@@ -6346,6 +6351,20 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6351
});
6352
}
6353
const before = yield* orchestrator.getThreadProjection(threadId);
6354
+ const source = before.runs[0]!;
6355
+ if (started) {
6356
+ yield* (yield* EventSink.EventSinkV2).write({
6357
+ events: [
6358
+ {
6359
+ id: EventId.make(`${threadId}:started`),
6360
+ type: "run.updated",
6361
+ threadId,
6362
+ occurredAt: yield* DateTime.now,
6363
+ payload: { ...source, status: "running", startedAt: yield* DateTime.now },
6364
+ },
6365
+ ],
6366
+ });
6367
+ }
6368
const queued = before.runs.filter((run) => run.status === "queued");
6369
assert.equal(queued.length, 2);
6370
yield* recovery.reconcile(trigger);
@@ -6356,12 +6375,33 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6375
const held = yield* orchestrator.getThreadProjection(threadId);
6376
assert.deepEqual(
6377
held.runs.map((run) => run.status),
6359
- ["cancelled", "queued", "queued"],
6378
+ [started ? "cancelled" : "queued", "queued", "queued"],
6379
);
6361
- for (const run of queued) {
6380
+ if (!started) {
6381
+ assert.deepInclude(
6382
+ held.runs.find((run) => run.id === source.id),
6383
+ {
6384
+ status: "queued",
6385
+ queueHeld: true,
6386
+ queuePosition: 1,
6387
+ },
6388
+ );
6389
+ }
6390
+ if (!started) {
6391
+ const prompt = held.turnItems.find(
6392
+ (item) => item.type === "user_message" && item.messageId === source.userMessageId,
6393
+ );
6394
+ assert.deepInclude(prompt, {
6395
+ inputIntent: "queued_turn",
6396
+ status: "pending",
6397
+ startedAt: null,
6398
+ completedAt: null,
6399
+ });
6400
+ }
6401
+ for (const [index, run] of queued.entries()) {
6402
assert.deepEqual(
6403
held.runs.find((row) => row.id === run.id),
6364
- { ...run, queueHeld: true },
6404
+ { ...run, queueHeld: true, ...(started ? {} : { queuePosition: index + 2 }) },
6405
);
6406
assert.deepEqual(
6407
held.messages.find((row) => row.id === run.userMessageId),
@@ -6374,21 +6414,23 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6414
assert.equal(held.nodes.find((row) => row.id === run.rootNodeId)?.status, "pending");
6415
}
6416
const first = queued[0]!;
6377
- const second = queued[1]!;
6378
- yield* orchestrator.dispatch({
6379
- type: "queued-run.edit",
6380
- commandId: CommandId.make(`${threadId}:edit`),
6381
- threadId,
6382
- runId: second.id,
6383
- text: "Edited second message",
6384
- });
6385
- yield* orchestrator.dispatch({
6386
- type: "queued-run.reorder",
6387
- commandId: CommandId.make(`${threadId}:reorder`),
6388
- threadId,
6389
- runId: second.id,
6390
- beforeRunId: first.id,
6391
- });
6417
+ const second = started ? queued[1]! : source;
6418
+ if (started) {
6419
+ yield* orchestrator.dispatch({
6420
+ type: "queued-run.edit",
6421
+ commandId: CommandId.make(`${threadId}:edit`),
6422
+ threadId,
6423
+ runId: second.id,
6424
+ text: "Edited second message",
6425
+ });
6426
+ yield* orchestrator.dispatch({
6427
+ type: "queued-run.reorder",
6428
+ commandId: CommandId.make(`${threadId}:reorder`),
6429
+ threadId,
6430
+ runId: second.id,
6431
+ beforeRunId: first.id,
6432
+ });
6433
+ }
6434
assert.equal(yield* orchestrator.resumeQueuedRuns, 0);
6435
const resume = {
6436
type: "queue.resume" as const,
@@ -6403,9 +6445,41 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6445
assert.isFalse(resumed.runs.some((run) => run.status === "queued" && run.queueHeld));
6446
assert.equal(
6447
resumed.messages.find((row) => row.id === second.userMessageId)?.text,
6406
- "Edited second message",
6448
+ started ? "Edited second message" : "Active",
6449
+ );
6450
+ if (!started) {
6451
+ assert.equal(
6452
+ resumed.attempts.find((row) => row.id === source.activeAttemptId)?.status,
6453
+ "pending",
6454
+ );
6455
+ assert.equal(
6456
+ resumed.nodes.find((row) => row.id === source.rootNodeId)?.status,
6457
+ "pending",
6458
+ );
6459
+ // Recovery command identities include the restart timestamp.
6460
+ yield* TestClock.adjust("1 millis");
6461
+ yield* recovery.reconcile("startup");
6462
+ yield* orchestrator.dispatch({
6463
+ ...resume,
6464
+ commandId: CommandId.make(`${threadId}:resume-again`),
6465
+ });
6466
+ const starts = yield* (yield* EffectOutbox.EffectOutboxV2).listByCommandId(
6467
+ CommandId.make(`command:system:start-queued:${source.id}`),
6468
+ );
6469
+ const pendingStarts = starts.filter((effect) => effect.status === "pending");
6470
+ assert.lengthOf(pendingStarts, 1);
6471
+ assert.deepEqual(pendingStarts[0]?.request, {
6472
+ type: "provider-turn.start",
6473
+ runId: source.id,
6474
+ });
6475
+ }
6476
+ const afterRetries = yield* orchestrator.getThreadProjection(threadId);
6477
+ assert.equal(
6478
+ afterRetries.runs.length,
6479
+ 3,
6480
+ "resume retries must not duplicate messages or runs",
6481
);
6408
- assert.equal(resumed.runs.length, 3, "resume retries must not duplicate messages or runs");
6482
+ assert.lengthOf(afterRetries.messages, before.messages.length);
6483
}),
6484
);
6485
apps/web/src/components/chat/QueuedRunsControl.tsx
+1
-1
@@ -255,7 +255,7 @@ export function QueuedRunsControl({
255
aria-posinset={position + 1}
256
aria-setsize={items.length}
257
aria-current={isEditing ? "true" : undefined}
258
- className="group/queued flex w-full flex-col items-end gap-1 transition-opacity duration-150 ease-out starting:opacity-0 motion-reduce:transition-none"
258
+ className="group/queued flex w-full flex-col items-end gap-1"
259
>
260
<QueuedMessage
261
environmentId={props.environmentId}