The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 02:00 UTC
fix(server): a usage-limit auto-resume survives a server restart (#31292)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
shivam committed
Nov 7, 2026 at 06:47 UTC
240446e2d9625600b0c841faf6bddf367bd26967
4 files changed
+155
-1
apps/server/src/orchestration-v2/Orchestrator.ts
+9
@@ -6279,6 +6279,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6279
...(command.restartContinuationOfRunId === undefined
6280
? {}
6281
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6282
+ ...(command.usageLimitContinuationOfRunId === undefined
6283
+ ? {}
6284
+ : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
6285
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
6286
...(freshContext ? { freshContext: true } : {}),
6287
};
@@ -6659,6 +6662,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6662
...(command.restartContinuationOfRunId === undefined
6663
? {}
6664
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
6665
+ ...(command.usageLimitContinuationOfRunId === undefined
6666
+ ? {}
6667
+ : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
6668
...(dispatchMode.type === "defer_start" && dispatchMode.workspaceStrategy !== undefined
6669
? { workspacePreparation: dispatchMode.workspaceStrategy }
6670
: {}),
@@ -7445,6 +7451,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7451
...(command.restartContinuationOfRunId === undefined
7452
? {}
7453
: { restartContinuationOfRunId: command.restartContinuationOfRunId }),
7454
+ ...(command.usageLimitContinuationOfRunId === undefined
7455
+ ? {}
7456
+ : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),
7457
...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),
7458
...wakeWorkStartedAt(projection.runs, command),
7459
};
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+59
@@ -442,6 +442,64 @@ export const make = Effect.gen(function* () {
442
},
443
});
444
}
445
+ const unsentResumeRunIds = runs.flatMap((run) =>
446
+ run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],
447
+ );
448
+ const limitRecovery = projection.thread.limitRecovery;
449
+ // The unsent resume spent the worker's request id, so a fresh one lets it send again. An
450
+ // untagged resume from an older server stays cancelled: its copy couldn't be hidden.
451
+ if (limitRecovery?.autoResume === true && unsentResumeRunIds.length > 0) {
452
+ events.push({
453
+ id: yield* allocateEventId(),
454
+ type: "thread.metadata-updated",
455
+ threadId: projection.thread.id,
456
+ occurredAt: now,
457
+ payload: {
458
+ ...projection.thread,
459
+ limitRecovery: { ...limitRecovery, requestId: commandId },
460
+ updatedAt: now,
461
+ },
462
+ });
463
+ }
464
+ // The worker sends the auto-resume again, so the unsent copy leaves the transcript the way
465
+ // cancelled queued input does.
466
+ const unsentResumePrompts =
467
+ unsentResumeRunIds.length === 0
468
+ ? []
469
+ : (yield* projections
470
+ .getThreadRecords(projection.thread.id, ["turnItems"], {
471
+ turnItemTypes: ["user_message"],
472
+ turnItemRunIds: unsentResumeRunIds,
473
+ })
474
+ .pipe(
475
+ Effect.mapError(
476
+ (cause) =>
477
+ new ProviderRuntimeRecoveryError({
478
+ operation: "read-projections",
479
+ threadId: projection.thread.id,
480
+ cause,
481
+ }),
482
+ ),
483
+ )).turnItems;
484
+ const unsentResumePromptIds = new Set(unsentResumePrompts.map((item) => item.id));
485
+ for (const item of unsentResumePrompts) {
486
+ if (item.type !== "user_message") continue;
487
+ const active =
488
+ item.status === "pending" || item.status === "running" || item.status === "waiting";
489
+ events.push({
490
+ id: yield* allocateEventId(),
491
+ type: "turn-item.updated",
492
+ threadId: projection.thread.id,
493
+ ...(item.runId === null ? {} : { runId: item.runId }),
494
+ occurredAt: now,
495
+ payload: {
496
+ ...item,
497
+ ...(active ? { status: "cancelled", completedAt: now } : {}),
498
+ inputIntent: "queued_turn",
499
+ updatedAt: now,
500
+ },
501
+ });
502
+ }
503
for (const question of questions) {
504
events.push({
505
id: yield* allocateEventId(),
@@ -552,6 +610,7 @@ export const make = Effect.gen(function* () {
610
for (const item of projection.turnItems.filter(
611
(candidate) =>
612
candidate.runId === run.id &&
613
+ !unsentResumePromptIds.has(candidate.id) &&
614
(candidate.nodeId === null || !messageRequestNodeIds.has(candidate.nodeId)) &&
615
!isAppOwnedDelegationItem(candidate) &&
616
// A link card waits on the user, not the run, and answers by message.
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+86
-1
@@ -1,6 +1,7 @@
1
import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";
2
import { SourceControlProviderKind } from "@t3tools/contracts";
3
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
4
+import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";
5
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
6
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
7
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -8136,6 +8137,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
8137
8138
it.effect.each([
8139
"resume",
8140
+ "resume-started",
8141
+ "resume-untagged",
8142
"queued-resume",
8143
"rearm",
8144
"snooze-race",
@@ -8614,17 +8617,97 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
8617
yield* orchestrator.dispatch(resume);
8618
}
8619
const after = yield* orchestrator.getThreadProjection(threadId);
8617
- if (scenario === "resume") {
8620
+ if (
8621
+ scenario === "resume" ||
8622
+ scenario === "resume-started" ||
8623
+ scenario === "resume-untagged"
8624
+ ) {
8625
assert.deepInclude(after.messages.at(-1), {
8626
text: "Continue where you left off.",
8627
createdBy: "agent",
8628
creationSource: "server",
8629
});
8630
+ const continuation = after.runs.at(-1)!;
8631
+ assert.equal(continuation.status, "starting");
8632
+ if (scenario === "resume-started") {
8633
+ const startedAt = yield* DateTime.now;
8634
+ yield* events.write({
8635
+ events: [
8636
+ {
8637
+ id: EventId.make("recovery:continuation-started:resume-started"),
8638
+ type: "run.updated",
8639
+ threadId,
8640
+ runId: continuation.id,
8641
+ occurredAt: startedAt,
8642
+ payload: { ...continuation, status: "running", startedAt },
8643
+ },
8644
+ ],
8645
+ });
8646
+ }
8647
+ if (scenario === "resume-untagged") {
8648
+ // A server older than the run tag stored the resume run without it.
8649
+ const { usageLimitContinuationOfRunId: _, ...untagged } = continuation;
8650
+ yield* events.write({
8651
+ events: [
8652
+ {
8653
+ id: EventId.make("recovery:continuation-untagged"),
8654
+ type: "run.updated",
8655
+ threadId,
8656
+ runId: continuation.id,
8657
+ occurredAt: yield* DateTime.now,
8658
+ payload: untagged,
8659
+ },
8660
+ ],
8661
+ });
8662
+ }
8663
+ yield* (yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService).reconcile(
8664
+ "startup",
8665
+ );
8666
+ const restartedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
8667
+ (thread) => thread.id === threadId,
8668
+ )!;
8669
+ const retry = limitRecoveryCommand(
8670
+ restartedShell,
8671
+ true,
8672
+ DateTime.toEpochMillis(yield* DateTime.now),
8673
+ );
8674
+ if (scenario === "resume-started") assert.isNull(retry);
8675
+ else assert(retry?.type === "message.dispatch");
8676
+ if (scenario === "resume") {
8677
+ assert.notEqual(retry!.commandId, resume!.commandId);
8678
+ } else {
8679
+ assert.equal(
8680
+ restartedShell.limitRecovery?.requestId,
8681
+ after.thread.limitRecovery?.requestId,
8682
+ );
8683
+ }
8684
+ if (retry !== null) {
8685
+ yield* orchestrator.dispatch(retry);
8686
+ yield* orchestrator.dispatch(retry);
8687
+ }
8688
+ const resumed = yield* orchestrator.getThreadProjection(threadId);
8689
+ const visible = createOrchestrationV2TurnItemVisibility({
8690
+ runs: resumed.runs,
8691
+ attempts: resumed.attempts,
8692
+ items: resumed.turnItems,
8693
+ });
8694
+ assert.deepEqual(
8695
+ resumed.turnItems.flatMap((item) =>
8696
+ item.type === "user_message" && visible(item) ? [item.runId] : [],
8697
+ ),
8698
+ [run.id, resumed.runs.at(-1)!.id],
8699
+ );
8700
+ assert.deepEqual(
8701
+ resumed.runs.map((row) => row.status),
8702
+ scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"],
8703
+ );
8704
}
8705
assert.lengthOf(
8706
after.runs,
8707
before.runs.length +
8708
(scenario === "resume" ||
8709
+ scenario === "resume-started" ||
8710
+ scenario === "resume-untagged" ||
8711
stopBeforeArm ||
8712
scenario === "queued-resume" ||
8713
scenario === "independent-patches" ||
@@ -8636,6 +8719,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
8719
after.messages,
8720
before.messages.length +
8721
(scenario === "resume" ||
8722
+ scenario === "resume-started" ||
8723
+ scenario === "resume-untagged" ||
8724
stopBeforeArm ||
8725
scenario === "queued-resume" ||
8726
scenario === "independent-patches" ||
packages/contracts/src/orchestrationV2.ts
+1
@@ -692,6 +692,7 @@ export const OrchestrationV2Run = Schema.Struct({
692
checkpointId: Schema.NullOr(CheckpointId),
693
contextHandoffId: Schema.NullOr(ContextHandoffId),
694
restartContinuationOfRunId: Schema.optional(RunId),
695
+ usageLimitContinuationOfRunId: Schema.optional(RunId),
696
delegatedTaskId: Schema.optional(NodeId),
697
workStartedAt: Schema.optional(Schema.DateTimeUtc),
698
restartCancelledBackgroundWork: Schema.optional(