@devries-cameron20 / t3code Mirror · read-only / commits / 737afe24bd
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 04:30 UTC

fix(server): retry unstarted runs after recovery (#30560)

Yash Singh committed Nov 3, 2026 at 20:54 UTC 737afe24bd20d1390f36fa6f5d4e7974dffead3c
5 files changed +233 -39
apps/server/src/orchestration-v2/Orchestrator.ts
+2 -1
@@ -2376,7 +2376,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2376 },
2377 })),
2378 {
2379 - id: `effect:${commandId}:provider-turn.start:${queuedRun.id}`,
2379 + // A restart can return this run to the queue after its previous start effect was cancelled.
2380 + id: `effect:${commandId}:provider-turn.start:${queuedRun.id}:${yield* idAllocator.allocate.event({ threadId, commandId })}`,
2381 commandId,
2382 threadId,
2383 request: { type: "provider-turn.start", runId: queuedRun.id },
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
+51 -7
@@ -501,7 +501,19 @@ it.effect("recovery closes a secret card's form but leaves a link card open", ()
501 }).pipe(Effect.provide(layer));
502 });
503
504 -it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart", (status) => {
504 +it.effect.each([
505 + "queued",
506 + "preparing",
507 + "starting",
508 + "starting-other-request",
509 + "starting-own-request",
510 + "starting-scheduled",
511 +] as const)("recovers $0 work after restart", (scenario) => {
512 + const status = scenario.startsWith("starting") ? "starting" : scenario;
513 + const terminalized =
514 + status === "preparing" ||
515 + scenario === "starting-own-request" ||
516 + scenario === "starting-scheduled";
517 const threadId = ThreadId.make("thread_queued_restart");
518 const runId = RunId.make("run_queued_restart");
519 const attemptId = RunAttemptId.make("attempt_queued_restart");
@@ -510,7 +522,18 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
522 null;
523 const projection = {
524 thread: { id: threadId },
513 - runtimeRequests: [],
525 + runtimeRequests: scenario.endsWith("request")
526 + ? [
527 + {
528 + id: RuntimeRequestId.make("pending-recovery-request"),
529 + nodeId:
530 + scenario === "starting-own-request" ? rootNodeId : NodeId.make("other-run-request"),
531 + kind: "user_input",
532 + status: "pending",
533 + responseCapability: { type: "message" },
534 + },
535 + ]
536 + : [],
537 providerSessions: [],
538 providerThreads: [],
539 providerTurns: [],
@@ -518,6 +541,9 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
541 {
542 id: runId,
543 status,
544 + startedAt: null,
545 + activeAttemptId: attemptId,
546 + userMessageId: MessageId.make("message_queued_restart"),
547 queuePosition: 1,
548 providerInstanceId: ProviderInstanceId.make("codex"),
549 ...(status === "preparing"
@@ -540,6 +566,15 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
566 },
567 ],
568 nodes: [
569 + ...(scenario === "starting-other-request"
570 + ? [
571 + {
572 + id: NodeId.make("other-run-request"),
573 + runId: RunId.make("earlier-run"),
574 + status: "waiting",
575 + },
576 + ]
577 + : []),
578 {
579 id: rootNodeId,
580 runId,
@@ -547,7 +582,16 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
582 },
583 ],
584 subagents: [],
550 - messages: [],
585 + messages:
586 + status === "starting"
587 + ? [
588 + {
589 + id: MessageId.make("message_queued_restart"),
590 + createdBy: "user",
591 + ...(scenario === "starting-scheduled" ? { scheduledTaskId: "scheduled-task" } : {}),
592 + },
593 + ]
594 + : [],
595 turnItems: [],
596 } as unknown as OrchestrationV2ThreadProjection;
597 const layer = ProviderRuntimeRecovery.layer.pipe(
@@ -579,7 +623,7 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
623 return Effect.gen(function* () {
624 const summary =
625 yield* (yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService).reconcile("startup");
582 - assert.equal(summary.terminalizedRuns, status === "preparing" ? 1 : 0);
626 + assert.equal(summary.terminalizedRuns, terminalized ? 1 : 0);
627 const command = committedInput;
628 assert.isNotNull(command);
629 if (command === null) return;
@@ -588,17 +632,17 @@ it.effect.each(["queued", "preparing"] as const)("recovers $0 work after restart
632 const nodeEvent = command.events.find((event) => event.type === "node.updated");
633 assert.equal(
634 runEvent?.type === "run.updated" ? runEvent.payload.status : null,
591 - status === "preparing" ? "cancelled" : "queued",
635 + terminalized ? "cancelled" : "queued",
636 );
637 assert.isEmpty(command.effects);
594 - if (status === "preparing") {
638 + if (terminalized) {
639 assert.equal(
640 attemptEvent?.type === "run-attempt.updated" ? attemptEvent.payload.status : null,
641 "cancelled",
642 );
643 assert.equal(
644 nodeEvent?.type === "node.updated" ? nodeEvent.payload.status : null,
601 - "cancelled",
645 + scenario === "starting-own-request" ? null : "cancelled",
646 );
647 } else {
648 assert.equal(runEvent?.type === "run.updated" ? runEvent.payload.queuePosition : null, 1);
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+81 -6
@@ -25,6 +25,7 @@ import * as ProjectionStore from "./ProjectionStore.ts";
25 import * as ServerSettings from "../serverSettings.ts";
26 import { restartContinuationRun } from "./RestartContinuation.ts";
27 import { isBufferedProviderNotification } from "./NotificationMailbox.ts";
28 +import { queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
29 import {
30 endOrphanedNativeSubagent,
31 endRunlessRootTurns,
@@ -210,7 +211,7 @@ export const make = Effect.gen(function* () {
211 stoppedThisPass: ReadonlyMap<string, RecoverySession> = new Map(),
212 ) {
213 const now = yield* DateTime.now;
213 - const runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
214 + let runs = [] as Array<OrchestrationV2ThreadProjection["runs"][number]>;
215 for (const run of nonterminalRuns(projection)) {
216 if (run.status === "waiting") {
217 const checkpointEffects = yield* outbox
@@ -325,13 +326,25 @@ export const make = Effect.gen(function* () {
326 cancelledBackgroundNativeIds.add(item.nativeItemRef.nativeId);
327 }
328 };
329 + const unstartedRuns = runs.filter(
330 + (run) =>
331 + run.status === "starting" &&
332 + neverStarted(run) &&
333 + !projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId) &&
334 + !projection.runtimeRequests.some(
335 + (request) =>
336 + request.status === "pending" &&
337 + projection.nodes.some((node) => node.id === request.nodeId && node.runId === run.id),
338 + ),
339 + );
340 const queuedRuns = projection.runs.filter((run) => run.status === "queued");
341 + const pendingRuns = [...unstartedRuns, ...queuedRuns];
342 const queuedMessages =
330 - queuedRuns.length === 0
343 + pendingRuns.length === 0
344 ? []
345 : (yield* projections
346 .getThreadRecords(projection.thread.id, ["messages"], {
334 - messageIds: queuedRuns.map((run) => run.userMessageId),
347 + messageIds: pendingRuns.map((run) => run.userMessageId),
348 })
349 .pipe(
350 Effect.mapError(
@@ -346,12 +359,69 @@ export const make = Effect.gen(function* () {
359 const bufferedMessageIds = new Set(
360 queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
361 );
349 - for (const run of queuedRuns) {
362 + const retryableMessageIds = new Set(
363 + queuedMessages
364 + .filter(
365 + (message) =>
366 + message.createdBy === "user" &&
367 + message.scheduledTaskId === undefined &&
368 + message.notification === undefined &&
369 + message.delegatedCompletion === undefined,
370 + )
371 + .map((message) => message.id),
372 + );
373 + // A saved user message the provider never accepted stays available through the held queue.
374 + const retryableRuns = unstartedRuns.filter((run) =>
375 + retryableMessageIds.has(run.userMessageId),
376 + );
377 + const retryableRunIds = new Set(retryableRuns.map((run) => run.id));
378 + const retryableItems =
379 + retryableRuns.length === 0
380 + ? []
381 + : (yield* projections
382 + .getThreadRecords(projection.thread.id, ["turnItems"], {
383 + turnItemTypes: ["user_message"],
384 + turnItemRunIds: retryableRuns.map((run) => run.id),
385 + })
386 + .pipe(
387 + Effect.mapError(
388 + (cause) =>
389 + new ProviderRuntimeRecoveryError({
390 + operation: "read-projections",
391 + threadId: projection.thread.id,
392 + cause,
393 + }),
394 + ),
395 + )).turnItems;
396 + for (const item of retryableItems) {
397 + if (item.type !== "user_message" || !retryableMessageIds.has(item.messageId)) continue;
398 + events.push({
399 + id: yield* allocateEventId(),
400 + type: "turn-item.updated",
401 + threadId: projection.thread.id,
402 + ...(item.runId === null ? {} : { runId: item.runId }),
403 + occurredAt: now,
404 + payload: {
405 + ...item,
406 + inputIntent: "queued_turn",
407 + status: "pending",
408 + startedAt: null,
409 + completedAt: null,
410 + updatedAt: now,
411 + },
412 + });
413 + }
414 + runs = runs.filter((run) => !retryableRunIds.has(run.id));
415 + const heldRuns = [
416 + ...retryableRuns,
417 + ...queuedRunsInDeliveryOrder({ runs: queuedRuns, messages: queuedMessages }),
418 + ];
419 + for (const [index, run] of heldRuns.entries()) {
420 if (bufferedMessageIds.has(run.userMessageId)) {
421 runs.push(run);
422 continue;
423 }
354 - if (run.queueHeld === true) continue;
424 + if (run.queueHeld === true && retryableRuns.length === 0) continue;
425 events.push({
426 id: yield* allocateEventId(),
427 type: "run.updated",
@@ -359,7 +429,12 @@ export const make = Effect.gen(function* () {
429 runId: run.id,
430 providerInstanceId: run.providerInstanceId,
431 occurredAt: now,
362 - payload: { ...run, queueHeld: true },
432 + payload: {
433 + ...run,
434 + status: "queued",
435 + queueHeld: true,
436 + ...(retryableRuns.length === 0 ? {} : { queuePosition: index + 1 }),
437 + },
438 });
439 }
440 for (const question of questions) {
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+98 -24
@@ -6310,13 +6310,18 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6310 }),
6311 );
6312
6313 - it.effect.each(["startup", "shutdown"] as const)(
6314 - "preserves and holds queued messages across %s until explicitly resumed",
6315 - (trigger) =>
6313 + it.effect.each([
6314 + { trigger: "startup", started: true },
6315 + { trigger: "shutdown", started: true },
6316 + { trigger: "startup", started: false },
6317 + { trigger: "shutdown", started: false },
6318 + ] as const)(
6319 + "preserves and holds queued messages across $trigger with started=$started until explicitly resumed",
6320 + ({ trigger, started }) =>
6321 Effect.gen(function* () {
6322 const orchestrator = yield* Orchestrator.OrchestratorV2;
6323 const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
6319 - const threadId = ThreadId.make(`queue-hold-${trigger}`);
6324 + const threadId = ThreadId.make(`queue-hold-${trigger}-${started}`);
6325 yield* orchestrator.dispatch({
6326 type: "thread.create",
6327 createdBy: "user",
@@ -6346,6 +6351,20 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6351 });
6352 }
6353 const before = yield* orchestrator.getThreadProjection(threadId);
6354 + const source = before.runs[0]!;
6355 + if (started) {
6356 + yield* (yield* EventSink.EventSinkV2).write({
6357 + events: [
6358 + {
6359 + id: EventId.make(`${threadId}:started`),
6360 + type: "run.updated",
6361 + threadId,
6362 + occurredAt: yield* DateTime.now,
6363 + payload: { ...source, status: "running", startedAt: yield* DateTime.now },
6364 + },
6365 + ],
6366 + });
6367 + }
6368 const queued = before.runs.filter((run) => run.status === "queued");
6369 assert.equal(queued.length, 2);
6370 yield* recovery.reconcile(trigger);
@@ -6356,12 +6375,33 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6375 const held = yield* orchestrator.getThreadProjection(threadId);
6376 assert.deepEqual(
6377 held.runs.map((run) => run.status),
6359 - ["cancelled", "queued", "queued"],
6378 + [started ? "cancelled" : "queued", "queued", "queued"],
6379 );
6361 - for (const run of queued) {
6380 + if (!started) {
6381 + assert.deepInclude(
6382 + held.runs.find((run) => run.id === source.id),
6383 + {
6384 + status: "queued",
6385 + queueHeld: true,
6386 + queuePosition: 1,
6387 + },
6388 + );
6389 + }
6390 + if (!started) {
6391 + const prompt = held.turnItems.find(
6392 + (item) => item.type === "user_message" && item.messageId === source.userMessageId,
6393 + );
6394 + assert.deepInclude(prompt, {
6395 + inputIntent: "queued_turn",
6396 + status: "pending",
6397 + startedAt: null,
6398 + completedAt: null,
6399 + });
6400 + }
6401 + for (const [index, run] of queued.entries()) {
6402 assert.deepEqual(
6403 held.runs.find((row) => row.id === run.id),
6364 - { ...run, queueHeld: true },
6404 + { ...run, queueHeld: true, ...(started ? {} : { queuePosition: index + 2 }) },
6405 );
6406 assert.deepEqual(
6407 held.messages.find((row) => row.id === run.userMessageId),
@@ -6374,21 +6414,23 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6414 assert.equal(held.nodes.find((row) => row.id === run.rootNodeId)?.status, "pending");
6415 }
6416 const first = queued[0]!;
6377 - const second = queued[1]!;
6378 - yield* orchestrator.dispatch({
6379 - type: "queued-run.edit",
6380 - commandId: CommandId.make(`${threadId}:edit`),
6381 - threadId,
6382 - runId: second.id,
6383 - text: "Edited second message",
6384 - });
6385 - yield* orchestrator.dispatch({
6386 - type: "queued-run.reorder",
6387 - commandId: CommandId.make(`${threadId}:reorder`),
6388 - threadId,
6389 - runId: second.id,
6390 - beforeRunId: first.id,
6391 - });
6417 + const second = started ? queued[1]! : source;
6418 + if (started) {
6419 + yield* orchestrator.dispatch({
6420 + type: "queued-run.edit",
6421 + commandId: CommandId.make(`${threadId}:edit`),
6422 + threadId,
6423 + runId: second.id,
6424 + text: "Edited second message",
6425 + });
6426 + yield* orchestrator.dispatch({
6427 + type: "queued-run.reorder",
6428 + commandId: CommandId.make(`${threadId}:reorder`),
6429 + threadId,
6430 + runId: second.id,
6431 + beforeRunId: first.id,
6432 + });
6433 + }
6434 assert.equal(yield* orchestrator.resumeQueuedRuns, 0);
6435 const resume = {
6436 type: "queue.resume" as const,
@@ -6403,9 +6445,41 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
6445 assert.isFalse(resumed.runs.some((run) => run.status === "queued" && run.queueHeld));
6446 assert.equal(
6447 resumed.messages.find((row) => row.id === second.userMessageId)?.text,
6406 - "Edited second message",
6448 + started ? "Edited second message" : "Active",
6449 + );
6450 + if (!started) {
6451 + assert.equal(
6452 + resumed.attempts.find((row) => row.id === source.activeAttemptId)?.status,
6453 + "pending",
6454 + );
6455 + assert.equal(
6456 + resumed.nodes.find((row) => row.id === source.rootNodeId)?.status,
6457 + "pending",
6458 + );
6459 + // Recovery command identities include the restart timestamp.
6460 + yield* TestClock.adjust("1 millis");
6461 + yield* recovery.reconcile("startup");
6462 + yield* orchestrator.dispatch({
6463 + ...resume,
6464 + commandId: CommandId.make(`${threadId}:resume-again`),
6465 + });
6466 + const starts = yield* (yield* EffectOutbox.EffectOutboxV2).listByCommandId(
6467 + CommandId.make(`command:system:start-queued:${source.id}`),
6468 + );
6469 + const pendingStarts = starts.filter((effect) => effect.status === "pending");
6470 + assert.lengthOf(pendingStarts, 1);
6471 + assert.deepEqual(pendingStarts[0]?.request, {
6472 + type: "provider-turn.start",
6473 + runId: source.id,
6474 + });
6475 + }
6476 + const afterRetries = yield* orchestrator.getThreadProjection(threadId);
6477 + assert.equal(
6478 + afterRetries.runs.length,
6479 + 3,
6480 + "resume retries must not duplicate messages or runs",
6481 );
6408 - assert.equal(resumed.runs.length, 3, "resume retries must not duplicate messages or runs");
6482 + assert.lengthOf(afterRetries.messages, before.messages.length);
6483 }),
6484 );
6485
apps/web/src/components/chat/QueuedRunsControl.tsx
+1 -1
@@ -255,7 +255,7 @@ export function QueuedRunsControl({
255 aria-posinset={position + 1}
256 aria-setsize={items.length}
257 aria-current={isEditing ? "true" : undefined}
258 - className="group/queued flex w-full flex-col items-end gap-1 transition-opacity duration-150 ease-out starting:opacity-0 motion-reduce:transition-none"
258 + className="group/queued flex w-full flex-col items-end gap-1"
259 >
260 <QueuedMessage
261 environmentId={props.environmentId}