The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 23:15 UTC
fix(server): stopped usage-limit threads no longer auto-resume (#21114)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
fixfon committed
Oct 20, 2026 at 08:04 UTC
60f2246358050fa70c7aee3cd685fefaaaaf81db
6 files changed
+196
-8
apps/server/src/orchestration-v2/Orchestrator.ts
+71
-1
@@ -27,6 +27,7 @@ import {
27
type ModelSelection,
28
OrchestrationV2Command,
29
type OrchestrationV2InternalCommand,
30
+ type OrchestrationV2LimitRecoveryUpdate,
31
type OrchestrationV2ServerCommand,
32
type ThreadPullRequestLink,
33
type ThreadPullRequestWatch,
@@ -312,6 +313,11 @@ export interface OrchestratorV2Shape {
313
readonly dispatch: (
314
command: OrchestrationV2ServerCommand,
315
) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
316
+ readonly autoArmLimitRecovery: (input: {
317
+ readonly commandId: CommandId;
318
+ readonly threadId: ThreadId;
319
+ readonly limitRecovery: OrchestrationV2LimitRecoveryUpdate;
320
+ }) => Effect.Effect<void, OrchestratorV2Error>;
321
readonly searchThreadStream: (
322
input: OrchestrationV2SearchThreadInput,
323
) => Stream.Stream<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
@@ -8978,7 +8984,49 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8984
readonly now: DateTime.Utc;
8985
}) {
8986
const emitEvent = emit(input.events, input.command);
8981
- const { thread, runs, subagents } = input.projection;
8987
+ const { runs, subagents } = input.projection;
8988
+ let { thread } = input.projection;
8989
+ let recovery = thread.limitRecovery;
8990
+ const latest = latestExecutedRun(runs);
8991
+ if (latest?.status === "failed") {
8992
+ const { turnItems } = yield* projectionStore
8993
+ .getThreadRecords(thread.id, ["turnItems"], {
8994
+ turnItemTypes: ["error"],
8995
+ turnItemRunIds: [latest.id],
8996
+ })
8997
+ .pipe(mapDispatchError(input.command));
8998
+ const failure = latestRootProviderFailure(latest, turnItems);
8999
+ if (
9000
+ failure?.class === "usage_limit" &&
9001
+ failure.resetAt != null &&
9002
+ Number.isFinite(Date.parse(failure.resetAt)) &&
9003
+ (recovery?.runId !== latest.id || recovery.resetAt !== failure.resetAt)
9004
+ ) {
9005
+ recovery = {
9006
+ runId: latest.id,
9007
+ resetAt: failure.resetAt,
9008
+ autoResume: false,
9009
+ snooze: false,
9010
+ };
9011
+ }
9012
+ }
9013
+ if (recovery && (recovery !== thread.limitRecovery || recovery.autoResume)) {
9014
+ thread = {
9015
+ ...thread,
9016
+ limitRecovery: {
9017
+ ...recovery,
9018
+ autoResume: false,
9019
+ requestId: input.command.commandId,
9020
+ },
9021
+ updatedAt: input.now,
9022
+ };
9023
+ yield* emitEvent({
9024
+ type: "thread.metadata-updated",
9025
+ threadId: thread.id,
9026
+ occurredAt: input.now,
9027
+ payload: thread,
9028
+ });
9029
+ }
9030
const queuedRuns = runs.filter((run) => run.status === "queued");
9031
const queuedMessages =
9032
queuedRuns.length === 0
@@ -11760,6 +11808,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11808
awaitRestartContinuation: (run) =>
11809
effectOutbox.awaitSettled(`effect:restart-continuation:${run.id}`).pipe(Effect.ignore),
11810
dispatch: dispatchWithReceipt,
11811
+ autoArmLimitRecovery: (input) =>
11812
+ threadDispatch.withLock(
11813
+ input.threadId,
11814
+ Effect.gen(function* () {
11815
+ const thread = yield* projectionStore
11816
+ .getThread(input.threadId)
11817
+ .pipe(
11818
+ Effect.mapError(
11819
+ (cause) => new OrchestratorProjectionError({ threadId: input.threadId, cause }),
11820
+ ),
11821
+ );
11822
+ // Stop or an explicit recovery choice can commit after the sweep reads its candidate.
11823
+ if (
11824
+ thread.limitRecovery?.runId === input.limitRecovery.runId &&
11825
+ thread.limitRecovery.resetAt === input.limitRecovery.resetAt
11826
+ )
11827
+ return;
11828
+ yield* dispatchWithReceiptEffect({ type: "thread.metadata.update", ...input });
11829
+ }),
11830
+ ),
11831
searchThreadStream: (input) =>
11832
projectionStore
11833
.searchThreadStream(input)
@@ -11909,6 +11977,8 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
11977
delegatedTaskResultPending: () => Effect.succeed(false),
11978
restartContinuationPending: () => Effect.succeed(false),
11979
awaitRestartContinuation: () => Effect.void,
11980
+ autoArmLimitRecovery: (input) =>
11981
+ Effect.fail(new OrchestratorProjectionError({ threadId: input.threadId })),
11982
dispatch: (command) =>
11983
Effect.fail(
11984
new OrchestratorDispatchError({
apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
+13
-1
@@ -5,6 +5,7 @@ import * as Layer from "effect/Layer";
5
import * as Scheduler from "../scheduling/Scheduler.ts";
6
import * as ServerSettings from "../serverSettings.ts";
7
import * as ProjectionStore from "./ProjectionStore.ts";
8
+import * as Orchestrator from "./Orchestrator.ts";
9
import * as ThreadManagement from "./ThreadManagementService.ts";
10
11
export function limitRecoveryCommand(
@@ -73,6 +74,7 @@ export function limitRecoveryCommand(
74
const makeSweep = Effect.gen(function* () {
75
const projections = yield* ProjectionStore.ProjectionStoreV2;
76
const threads = yield* ThreadManagement.ThreadManagementService;
77
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
78
const settings = yield* ServerSettings.ServerSettingsService;
79
return Effect.fn("UsageLimitRecoveryWorker.sweep")(function* () {
80
const preferences = yield* settings.getSettings;
@@ -91,7 +93,17 @@ const makeSweep = Effect.gen(function* () {
93
preferences.snoozeLimitedThreads,
94
);
95
if (command === null) continue;
94
- yield* threads.dispatch(command).pipe(
96
+ yield* Effect.gen(function* () {
97
+ if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
98
+ yield* orchestrator.autoArmLimitRecovery({
99
+ commandId: command.commandId,
100
+ threadId: command.threadId,
101
+ limitRecovery: command.limitRecovery,
102
+ });
103
+ } else {
104
+ yield* threads.dispatch(command);
105
+ }
106
+ }).pipe(
107
Effect.catchCause((cause) =>
108
Effect.logWarning("orchestration-v2.limit-recovery.dispatch-failed", {
109
threadId: thread.id,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+101
-3
@@ -6198,6 +6198,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6198
"independent-patches",
6199
"expired-snooze",
6200
"wake",
6201
+ "stop",
6202
+ "stop-before-arm",
6203
+ "stop-before-arm-mismatched-recovery",
6204
] as const)("guards a scheduled usage-limit continuation against %s", (scenario) =>
6205
Effect.gen(function* () {
6206
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -6354,8 +6357,68 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6357
].includes(scenario);
6358
const autoResume = scenario !== "snooze-only" && scenario !== "wake";
6359
const arm = limitRecoveryCommand(shell, autoResume, DateTime.toEpochMillis(now), snooze);
6357
- assert.isNotNull(arm);
6358
- yield* orchestrator.dispatch(arm!);
6360
+ assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);
6361
+ const automaticArm = {
6362
+ commandId: arm.commandId,
6363
+ threadId,
6364
+ limitRecovery: arm.limitRecovery,
6365
+ };
6366
+ const stopBeforeArm =
6367
+ scenario === "stop-before-arm" || scenario === "stop-before-arm-mismatched-recovery";
6368
+ if (stopBeforeArm) {
6369
+ if (scenario === "stop-before-arm-mismatched-recovery") {
6370
+ const current = yield* orchestrator.getThreadProjection(threadId);
6371
+ yield* events.write({
6372
+ events: [
6373
+ {
6374
+ id: EventId.make(`recovery:older-window:${scenario}`),
6375
+ type: "thread.metadata-updated",
6376
+ threadId,
6377
+ occurredAt: now,
6378
+ payload: {
6379
+ ...current.thread,
6380
+ limitRecovery: {
6381
+ runId: run.id,
6382
+ resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 2 })),
6383
+ autoResume: true,
6384
+ snooze: false,
6385
+ },
6386
+ },
6387
+ },
6388
+ ],
6389
+ });
6390
+ }
6391
+ yield* orchestrator.dispatch({
6392
+ type: "thread.stop",
6393
+ commandId: CommandId.make(`recovery:stop:${scenario}`),
6394
+ threadId,
6395
+ });
6396
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
6397
+ assert.isUndefined(
6398
+ (yield* projections.getLimitRecoveryCandidates({
6399
+ now,
6400
+ autoResume: true,
6401
+ snooze: false,
6402
+ })).find((row) => row.id === threadId),
6403
+ );
6404
+ const stoppedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6405
+ (thread) => thread.id === threadId,
6406
+ )!;
6407
+ assert.isNull(limitRecoveryCommand(stoppedShell, true, DateTime.toEpochMillis(now)));
6408
+ yield* orchestrator.autoArmLimitRecovery(automaticArm);
6409
+ assert.deepEqual((yield* orchestrator.getThreadProjection(threadId)).thread.limitRecovery, {
6410
+ runId: run.id,
6411
+ resetAt,
6412
+ autoResume: false,
6413
+ snooze: false,
6414
+ requestId: CommandId.make(`recovery:stop:${scenario}`),
6415
+ });
6416
+ }
6417
+ const armCommand = stopBeforeArm
6418
+ ? { ...arm, commandId: CommandId.make(`recovery:explicit-rearm:${scenario}`) }
6419
+ : arm;
6420
+ if (stopBeforeArm) yield* orchestrator.dispatch(armCommand);
6421
+ else yield* orchestrator.autoArmLimitRecovery(automaticArm);
6422
let armedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6423
(thread) => thread.id === threadId,
6424
)!;
@@ -6364,7 +6427,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6427
resetAt,
6428
autoResume,
6429
snooze,
6367
- requestId: arm!.commandId,
6430
+ requestId: armCommand.commandId,
6431
});
6432
if (snooze)
6433
assert.equal(DateTime.toEpochMillis(armedShell.snoozedUntil!), Date.parse(resetAt));
@@ -6568,6 +6631,17 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6631
commandId: CommandId.make(`recovery:settle:${scenario}`),
6632
threadId,
6633
});
6634
+ if (scenario === "stop") {
6635
+ yield* orchestrator.dispatch({
6636
+ type: "thread.stop",
6637
+ commandId: CommandId.make(`recovery:stop:${scenario}`),
6638
+ threadId,
6639
+ });
6640
+ const stoppedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6641
+ (thread) => thread.id === threadId,
6642
+ )!;
6643
+ assert.deepInclude(stoppedShell.limitRecovery, { runId: run.id, autoResume: false });
6644
+ }
6645
if (scenario === "replacement") {
6646
const current = yield* orchestrator.getThreadProjection(threadId);
6647
const error = current.turnItems.find((item) => item.type === "error")!;
@@ -6626,6 +6700,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6700
after.runs,
6701
before.runs.length +
6702
(scenario === "resume" ||
6703
+ stopBeforeArm ||
6704
scenario === "queued-resume" ||
6705
scenario === "snooze-resume" ||
6706
scenario === "wake-preserve-resume" ||
@@ -6638,6 +6713,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6713
after.messages,
6714
before.messages.length +
6715
(scenario === "resume" ||
6716
+ stopBeforeArm ||
6717
scenario === "queued-resume" ||
6718
scenario === "snooze-resume" ||
6719
scenario === "wake-preserve-resume" ||
@@ -6646,6 +6722,28 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6722
? 1
6723
: 0),
6724
);
6725
+ if (scenario === "stop") {
6726
+ assert.deepInclude(after.thread.limitRecovery, { runId: run.id, autoResume: false });
6727
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
6728
+ const sweep = { now: yield* DateTime.now, autoResume: true, snooze: false };
6729
+ assert.isUndefined(
6730
+ (yield* projections.getLimitRecoveryCandidates(sweep)).find((row) => row.id === threadId),
6731
+ );
6732
+ yield* orchestrator.dispatch({
6733
+ type: "thread.metadata.update",
6734
+ commandId: CommandId.make("recovery:rearm-after-stop"),
6735
+ threadId,
6736
+ limitRecovery: { runId: run.id, resetAt, autoResume: true },
6737
+ });
6738
+ const rearmed = (yield* projections.getLimitRecoveryCandidates(sweep)).find(
6739
+ (row) => row.id === threadId,
6740
+ )!;
6741
+ assert.isDefined(rearmed);
6742
+ const freshResume = limitRecoveryCommand(rearmed, true, DateTime.toEpochMillis(sweep.now));
6743
+ assert.isNotNull(freshResume);
6744
+ yield* orchestrator.dispatch(freshResume!);
6745
+ assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 2);
6746
+ }
6747
if (scenario === "queued-resume") {
6748
const held = after.runs[1]!;
6749
assert.equal(held.status, "queued");
apps/server/src/orchestration-v2/runtimeLayer.ts
+7
-1
@@ -368,7 +368,13 @@ export const layerProduction = Layer.mergeAll(
368
layerScheduledTaskProvided,
369
layerSecretRequestsProvided,
370
UsageLimitRecoveryWorker.layer.pipe(
371
- Layer.provide(Layer.mergeAll(ProjectionStore.layer, layerThreadManagementProvided)),
371
+ Layer.provide(
372
+ Layer.mergeAll(
373
+ ProjectionStore.layer,
374
+ layerThreadManagementProvided,
375
+ layerOrchestratorProvided,
376
+ ),
377
+ ),
378
),
379
layerProviderContinuationWorkerProvided,
380
layerAgentSessionImporterProvided,
apps/server/src/scheduling/Scheduler.integration.test.ts
+2
@@ -19,6 +19,7 @@ import * as SqlClient from "effect/sql/SqlClient";
19
import * as TestClock from "effect/testing/TestClock";
20
21
import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
22
+import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
23
import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
24
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
25
import * as UsageLimitRecoveryWorker from "../orchestration-v2/UsageLimitRecoveryWorker.ts";
@@ -89,6 +90,7 @@ it.effect.each(["on time", "after restart"])(
90
const receipts = yield* Queue.unbounded<"task" | "retry">();
91
const layerDependencies = Layer.mergeAll(
92
NodeCrypto.layer,
93
+ Layer.mock(Orchestrator.OrchestratorV2)({}),
94
Layer.mock(ThreadLaunchService.ThreadLaunchService)({
95
launch: () =>
96
Queue.offer(receipts, "task").pipe(
docs/user/thread-sidebar.md
+2
-2
@@ -206,8 +206,8 @@ continuation. You can cancel it from the thread. Enable **Auto-resume limited
206
threads** in **Settings → General** on web and desktop, or **Settings → Thread
207
behavior** on mobile, to schedule limit stops by default.
208
The environment must be running when the reset arrives; it resumes overdue
209
-continuations after a restart. Sending a new message, archiving, or settling the
210
-thread prevents a pending continuation from starting.
209
+continuations after a restart. Stopping the thread, sending a new message,
210
+archiving, or settling it prevents a pending continuation from starting.
211
212
Choose **Snooze until reset** to hide the thread until its allowance returns.
213
Snooze and auto-resume are independent: snooze alone wakes the thread without