@devries-cameron20 / t3code-1 / commits / 240446e2d9
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a usage-limit auto-resume survives a server restart (#31292)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

shivam committed Nov 7, 2026 at 06:47 UTC 240446e2d9625600b0c841faf6bddf367bd26967
4 files changed +155 -1
apps/server/src/orchestration-v2/Orchestrator.ts
+9
@@ -6279,6 +6279,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6279 ...(command.restartContinuationOfRunId === undefined
6280 ? {}
6281 : { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6282 + ...(command.usageLimitContinuationOfRunId === undefined
6283 + ? {}
6284 + : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
6285 ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
6286 ...(freshContext ? { freshContext: true } : {}),
6287 };
@@ -6659,6 +6662,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6662 ...(command.restartContinuationOfRunId === undefined
6663 ? {}
6664 : { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6665 + ...(command.usageLimitContinuationOfRunId === undefined
6666 + ? {}
6667 + : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
6668 ...(dispatchMode.type === "defer_start" && dispatchMode.workspaceStrategy !== undefined
6669 ? { workspacePreparation: dispatchMode.workspaceStrategy }
6670 : {}),
@@ -7445,6 +7451,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7451 ...(command.restartContinuationOfRunId === undefined
7452 ? {}
7453 : { restartContinuationOfRunId: command.restartContinuationOfRunId }),
7454 + ...(command.usageLimitContinuationOfRunId === undefined
7455 + ? {}
7456 + : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
7457 ...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
7458 ...wakeWorkStartedAt(projection.runs, command),
7459 };
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+59
@@ -442,6 +442,64 @@ export const make = Effect.gen(function* () {
442 },
443 });
444 }
445 + const unsentResumeRunIds = runs.flatMap((run) =>
446 + run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],
447 + );
448 + const limitRecovery = projection.thread.limitRecovery;
449 + // The unsent resume spent the worker's request id, so a fresh one lets it send again. An
450 + // untagged resume from an older server stays cancelled: its copy couldn't be hidden.
451 + if (limitRecovery?.autoResume === true && unsentResumeRunIds.length > 0) {
452 + events.push({
453 + id: yield* allocateEventId(),
454 + type: "thread.metadata-updated",
455 + threadId: projection.thread.id,
456 + occurredAt: now,
457 + payload: {
458 + ...projection.thread,
459 + limitRecovery: { ...limitRecovery, requestId: commandId },
460 + updatedAt: now,
461 + },
462 + });
463 + }
464 + // The worker sends the auto-resume again, so the unsent copy leaves the transcript the way
465 + // cancelled queued input does.
466 + const unsentResumePrompts =
467 + unsentResumeRunIds.length === 0
468 + ? []
469 + : (yield* projections
470 + .getThreadRecords(projection.thread.id, ["turnItems"], {
471 + turnItemTypes: ["user_message"],
472 + turnItemRunIds: unsentResumeRunIds,
473 + })
474 + .pipe(
475 + Effect.mapError(
476 + (cause) =>
477 + new ProviderRuntimeRecoveryError({
478 + operation: "read-projections",
479 + threadId: projection.thread.id,
480 + cause,
481 + }),
482 + ),
483 + )).turnItems;
484 + const unsentResumePromptIds = new Set(unsentResumePrompts.map((item) => item.id));
485 + for (const item of unsentResumePrompts) {
486 + if (item.type !== "user_message") continue;
487 + const active =
488 + item.status === "pending" || item.status === "running" || item.status === "waiting";
489 + events.push({
490 + id: yield* allocateEventId(),
491 + type: "turn-item.updated",
492 + threadId: projection.thread.id,
493 + ...(item.runId === null ? {} : { runId: item.runId }),
494 + occurredAt: now,
495 + payload: {
496 + ...item,
497 + ...(active ? { status: "cancelled", completedAt: now } : {}),
498 + inputIntent: "queued_turn",
499 + updatedAt: now,
500 + },
501 + });
502 + }
503 for (const question of questions) {
504 events.push({
505 id: yield* allocateEventId(),
@@ -552,6 +610,7 @@ export const make = Effect.gen(function* () {
610 for (const item of projection.turnItems.filter(
611 (candidate) =>
612 candidate.runId === run.id &&
613 + !unsentResumePromptIds.has(candidate.id) &&
614 (candidate.nodeId === null || !messageRequestNodeIds.has(candidate.nodeId)) &&
615 !isAppOwnedDelegationItem(candidate) &&
616 // A link card waits on the user, not the run, and answers by message.
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+86 -1
@@ -1,6 +1,7 @@
1 import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";
2 import { SourceControlProviderKind } from "@t3tools/contracts";
3 import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
4 +import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";
5 import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
6 import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
7 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -8136,6 +8137,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
8137
8138 it.effect.each([
8139 "resume",
8140 + "resume-started",
8141 + "resume-untagged",
8142 "queued-resume",
8143 "rearm",
8144 "snooze-race",
@@ -8614,17 +8617,97 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
8617 yield* orchestrator.dispatch(resume);
8618 }
8619 const after = yield* orchestrator.getThreadProjection(threadId);
8617 - if (scenario === "resume") {
8620 + if (
8621 + scenario === "resume" ||
8622 + scenario === "resume-started" ||
8623 + scenario === "resume-untagged"
8624 + ) {
8625 assert.deepInclude(after.messages.at(-1), {
8626 text: "Continue where you left off.",
8627 createdBy: "agent",
8628 creationSource: "server",
8629 });
8630 + const continuation = after.runs.at(-1)!;
8631 + assert.equal(continuation.status, "starting");
8632 + if (scenario === "resume-started") {
8633 + const startedAt = yield* DateTime.now;
8634 + yield* events.write({
8635 + events: [
8636 + {
8637 + id: EventId.make("recovery:continuation-started:resume-started"),
8638 + type: "run.updated",
8639 + threadId,
8640 + runId: continuation.id,
8641 + occurredAt: startedAt,
8642 + payload: { ...continuation, status: "running", startedAt },
8643 + },
8644 + ],
8645 + });
8646 + }
8647 + if (scenario === "resume-untagged") {
8648 + // A server older than the run tag stored the resume run without it.
8649 + const { usageLimitContinuationOfRunId: _, ...untagged } = continuation;
8650 + yield* events.write({
8651 + events: [
8652 + {
8653 + id: EventId.make("recovery:continuation-untagged"),
8654 + type: "run.updated",
8655 + threadId,
8656 + runId: continuation.id,
8657 + occurredAt: yield* DateTime.now,
8658 + payload: untagged,
8659 + },
8660 + ],
8661 + });
8662 + }
8663 + yield* (yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService).reconcile(
8664 + "startup",
8665 + );
8666 + const restartedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
8667 + (thread) => thread.id === threadId,
8668 + )!;
8669 + const retry = limitRecoveryCommand(
8670 + restartedShell,
8671 + true,
8672 + DateTime.toEpochMillis(yield* DateTime.now),
8673 + );
8674 + if (scenario === "resume-started") assert.isNull(retry);
8675 + else assert(retry?.type === "message.dispatch");
8676 + if (scenario === "resume") {
8677 + assert.notEqual(retry!.commandId, resume!.commandId);
8678 + } else {
8679 + assert.equal(
8680 + restartedShell.limitRecovery?.requestId,
8681 + after.thread.limitRecovery?.requestId,
8682 + );
8683 + }
8684 + if (retry !== null) {
8685 + yield* orchestrator.dispatch(retry);
8686 + yield* orchestrator.dispatch(retry);
8687 + }
8688 + const resumed = yield* orchestrator.getThreadProjection(threadId);
8689 + const visible = createOrchestrationV2TurnItemVisibility({
8690 + runs: resumed.runs,
8691 + attempts: resumed.attempts,
8692 + items: resumed.turnItems,
8693 + });
8694 + assert.deepEqual(
8695 + resumed.turnItems.flatMap((item) =>
8696 + item.type === "user_message" && visible(item) ? [item.runId] : [],
8697 + ),
8698 + [run.id, resumed.runs.at(-1)!.id],
8699 + );
8700 + assert.deepEqual(
8701 + resumed.runs.map((row) => row.status),
8702 + scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"],
8703 + );
8704 }
8705 assert.lengthOf(
8706 after.runs,
8707 before.runs.length +
8708 (scenario === "resume" ||
8709 + scenario === "resume-started" ||
8710 + scenario === "resume-untagged" ||
8711 stopBeforeArm ||
8712 scenario === "queued-resume" ||
8713 scenario === "independent-patches" ||
@@ -8636,6 +8719,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
8719 after.messages,
8720 before.messages.length +
8721 (scenario === "resume" ||
8722 + scenario === "resume-started" ||
8723 + scenario === "resume-untagged" ||
8724 stopBeforeArm ||
8725 scenario === "queued-resume" ||
8726 scenario === "independent-patches" ||
packages/contracts/src/orchestrationV2.ts
+1
@@ -692,6 +692,7 @@ export const OrchestrationV2Run = Schema.Struct({
692 checkpointId: Schema.NullOr(CheckpointId),
693 contextHandoffId: Schema.NullOr(ContextHandoffId),
694 restartContinuationOfRunId: Schema.optional(RunId),
695 + usageLimitContinuationOfRunId: Schema.optional(RunId),
696 delegatedTaskId: Schema.optional(NodeId),
697 workStartedAt: Schema.optional(Schema.DateTimeUtc),
698 restartCancelledBackgroundWork: Schema.optional(