The best way to control Claude Code, Codex and more from any computer or phone
fix(server): cleanup keeps worktrees waiting for a usage-limit reset (#24689)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>
Victor Vogel committed
Oct 31, 2026 at 03:47 UTC
d7d5d7c79b704ea28d1e10f7b9bcc7bd525cd771
2 files changed
+151
-13
apps/server/src/storageCleanup.test.ts
+103
-5
@@ -112,14 +112,16 @@ describe("V2 storage cleanup eligibility", () => {
112
const candidate = () => shell({ branch: "feature", worktreePath: "/worktrees/feature" });
113
114
it("allows an idle worktree and rejects the project checkout", () => {
115
- expect(storageCleanupThreadIdle(candidate(), NOW_MS)).toBe(true);
116
- expect(storageCleanupThreadIdle(shell(), NOW_MS)).toBe(false);
115
+ expect(storageCleanupThreadIdle(candidate(), NOW_MS, DEFAULT_SERVER_SETTINGS)).toBe(true);
116
+ expect(storageCleanupThreadIdle(shell(), NOW_MS, DEFAULT_SERVER_SETTINGS)).toBe(false);
117
});
118
119
it.each(["running", "starting", "preparing", "waiting", "queued"] as const)(
120
"retains a worktree while its thread is %s",
121
(status) => {
122
- expect(storageCleanupThreadIdle(candidateWithStatus(status), NOW_MS)).toBe(false);
122
+ expect(
123
+ storageCleanupThreadIdle(candidateWithStatus(status), NOW_MS, DEFAULT_SERVER_SETTINGS),
124
+ ).toBe(false);
125
},
126
);
127
@@ -133,6 +135,7 @@ describe("V2 storage cleanup eligibility", () => {
135
pendingBackgroundTasks: [{ taskId: "task-1", kind: "command" }],
136
},
137
NOW_MS,
138
+ DEFAULT_SERVER_SETTINGS,
139
),
140
).toBe(false);
141
},
@@ -152,6 +155,7 @@ describe("V2 storage cleanup eligibility", () => {
155
},
156
},
157
NOW_MS,
158
+ DEFAULT_SERVER_SETTINGS,
159
),
160
).toBe(false);
161
},
@@ -159,16 +163,83 @@ describe("V2 storage cleanup eligibility", () => {
163
164
it("retains an active run even if the shell status is idle", () => {
165
expect(
162
- storageCleanupThreadIdle({ ...candidate(), activeRunId: RunId.make("run") }, NOW_MS),
166
+ storageCleanupThreadIdle(
167
+ { ...candidate(), activeRunId: RunId.make("run") },
168
+ NOW_MS,
169
+ DEFAULT_SERVER_SETTINGS,
170
+ ),
171
).toBe(false);
172
});
173
174
it("retains a queued prompt before the new run has been projected", () => {
175
expect(
168
- storageCleanupThreadIdle({ ...candidate(), latestUserMessageAt: at(-1_000) }, NOW_MS),
176
+ storageCleanupThreadIdle(
177
+ { ...candidate(), latestUserMessageAt: at(-1_000) },
178
+ NOW_MS,
179
+ DEFAULT_SERVER_SETTINGS,
180
+ ),
181
).toBe(false);
182
});
183
184
+ it.each([
185
+ { name: "snoozed until a future reset", snooze: true, resetIn: DAY_MS, idle: false },
186
+ { name: "set to auto-resume", autoResume: true, resetIn: DAY_MS, idle: false },
187
+ { name: "with recovery turned off", resetIn: DAY_MS, idle: true },
188
+ { name: "auto-resuming after the reset", autoResume: true, resetIn: -1_000, idle: false },
189
+ { name: "snoozed past its reset", snooze: true, resetIn: -1_000, idle: true },
190
+ {
191
+ name: "settled with an auto-resume pending",
192
+ autoResume: true,
193
+ settled: true,
194
+ resetIn: -1_000,
195
+ idle: true,
196
+ },
197
+ { name: "not yet armed by the recovery worker", armed: false, resetIn: DAY_MS, idle: false },
198
+ {
199
+ name: "not yet armed, auto-resume after reset",
200
+ armed: false,
201
+ autoResumeSetting: true,
202
+ resetIn: -1_000,
203
+ idle: false,
204
+ },
205
+ {
206
+ name: "not armed with recovery settings off",
207
+ armed: false,
208
+ settingsOff: true,
209
+ resetIn: DAY_MS,
210
+ idle: true,
211
+ },
212
+ ])(
213
+ "treats a usage-limited thread $name as idle: $idle",
214
+ ({
215
+ autoResume = false,
216
+ snooze = false,
217
+ armed = true,
218
+ autoResumeSetting = false,
219
+ settled,
220
+ settingsOff,
221
+ resetIn,
222
+ idle,
223
+ }) => {
224
+ const runId = RunId.make("run-1");
225
+ const resetAt = DateTime.formatIso(at(resetIn));
226
+ expect(
227
+ storageCleanupThreadIdle(
228
+ {
229
+ ...candidateWithStatus("failed"),
230
+ latestRunId: runId,
231
+ lastErrorClass: "usage_limit",
232
+ usageLimitResetAt: resetAt,
233
+ settledOverride: settled ? "settled" : null,
234
+ limitRecovery: armed ? { runId, resetAt, autoResume, snooze } : null,
235
+ },
236
+ NOW_MS,
237
+ { autoResumeLimitedThreads: autoResumeSetting, snoozeLimitedThreads: !settingsOff },
238
+ ),
239
+ ).toBe(idle);
240
+ },
241
+ );
242
+
243
it("uses V2 run activity instead of metadata refreshes for retention", () => {
244
const thread = candidate();
245
const runTime = at(-3 * DAY_MS);
@@ -2283,6 +2354,33 @@ describe("storage cleanup reports and local file policies", () => {
2354
});
2355
2356
describe("settled worktree rechecks", () => {
2357
+ it.live.each(["autoResumeLimitedThreads", "snoozeLimitedThreads"] as const)(
2358
+ "keeps a usage-limited checkout when %s is enabled during measurement",
2359
+ (preference) =>
2360
+ runCleanupTest(
2361
+ Effect.gen(function* () {
2362
+ const fixture = yield* pausedMeasurementFixture();
2363
+ fixture.setSettings({ autoResumeLimitedThreads: false, snoozeLimitedThreads: false });
2364
+ fixture.setThreads([
2365
+ shell({
2366
+ branch: "feature",
2367
+ worktreePath: fixture.worktree,
2368
+ status: "failed",
2369
+ latestRunId: RunId.make("limited-run"),
2370
+ lastErrorClass: "usage_limit",
2371
+ usageLimitResetAt: DateTime.formatIso(at(DAY_MS)),
2372
+ }),
2373
+ ]);
2374
+ const run = yield* fixture.service.runNow.pipe(Effect.forkChild);
2375
+ yield* Deferred.await(fixture.measuring);
2376
+ fixture.setSettings({ [preference]: true });
2377
+ yield* Deferred.succeed(fixture.resume, undefined);
2378
+ const report = yield* Fiber.join(run);
2379
+ expect(report.entries[0]?.outcome).toBe("kept");
2380
+ expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true);
2381
+ }),
2382
+ ),
2383
+ );
2384
it.live.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])(
2385
"keeps a checkout changed during measurement (%j)",
2386
(change) =>
apps/server/src/storageCleanup.ts
+48
-8
@@ -115,7 +115,16 @@ function sameProjectWorktreePolicies(left: ServerSettings, right: ServerSettings
115
);
116
}
117
118
-export function storageCleanupThreadIdle(thread: OrchestrationV2ThreadShell, now: number): boolean {
118
+type LimitRecoverySettings = Pick<
119
+ ServerSettings,
120
+ "autoResumeLimitedThreads" | "snoozeLimitedThreads"
121
+>;
122
+
123
+export function storageCleanupThreadIdle(
124
+ thread: OrchestrationV2ThreadShell,
125
+ now: number,
126
+ settings: LimitRecoverySettings,
127
+): boolean {
128
return (
129
thread.worktreePath !== null &&
130
thread.activeRunId === null &&
@@ -127,10 +136,41 @@ export function storageCleanupThreadIdle(thread: OrchestrationV2ThreadShell, now
136
thread.status === "rolled_back") &&
137
(thread.pendingBackgroundTasks?.length ?? 0) === 0 &&
138
thread.pendingRuntimeRequest === null &&
130
- !threadHasQueuedTurnStart(thread, now)
139
+ !threadHasQueuedTurnStart(thread, now) &&
140
+ !heldForLimitRecovery(thread, now, settings)
141
);
142
}
143
144
+// A run held for its usage-limit reset has its work still ahead of it, and a
145
+// fresh worktree would otherwise look unchanged and be removed before it runs.
146
+function heldForLimitRecovery(
147
+ thread: OrchestrationV2ThreadShell,
148
+ now: number,
149
+ settings: LimitRecoverySettings,
150
+): boolean {
151
+ if (
152
+ thread.status !== "failed" ||
153
+ thread.lastErrorClass !== "usage_limit" ||
154
+ !thread.usageLimitResetAt ||
155
+ thread.archivedAt !== null ||
156
+ thread.settledOverride === "settled"
157
+ )
158
+ return false;
159
+ const resetMs = Date.parse(thread.usageLimitResetAt);
160
+ const recovery = thread.limitRecovery;
161
+ if (recovery?.runId === thread.latestRunId && recovery.resetAt === thread.usageLimitResetAt)
162
+ // An auto-resume holds until its continuation replaces the failed run.
163
+ return recovery.autoResume || (recovery.snooze === true && resetMs > now);
164
+ // The recovery worker arms this failure on a later tick; hold it until then.
165
+ // Like the worker, skip a reset that had already passed when the run ended.
166
+ if (
167
+ !Number.isFinite(resetMs) ||
168
+ resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt)
169
+ )
170
+ return false;
171
+ return settings.autoResumeLimitedThreads || (settings.snoozeLimitedThreads && resetMs > now);
172
+}
173
+
174
/** PR metadata refreshes must not reset the inactivity clock. */
175
export function storageCleanupActivityAt(thread: OrchestrationV2ThreadShell): number {
176
return Math.max(
@@ -800,7 +840,7 @@ const make = Effect.gen(function* () {
840
if (project === undefined) return keep("project is unavailable");
841
if (
842
thread.branch === null ||
803
- !group.every((member) => storageCleanupThreadIdle(member, now))
843
+ !group.every((member) => storageCleanupThreadIdle(member, now, serverSettings))
844
)
845
return keep(waitForStop("thread is running or has pending work"));
846
if (group.some((member) => member.branch !== thread.branch))
@@ -969,7 +1009,7 @@ const make = Effect.gen(function* () {
1009
shell.branch !== thread.branch ||
1010
!Equal.equals(shell.settledAt, previous.settledAt) ||
1011
(shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) ||
972
- !storageCleanupThreadIdle(shell, now) ||
1012
+ !storageCleanupThreadIdle(shell, now, serverSettings) ||
1013
path.resolve(shell.worktreePath!) !== worktreePath ||
1014
storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous)
1015
);
@@ -1009,11 +1049,11 @@ const make = Effect.gen(function* () {
1049
return keep("commit changed since check");
1050
const finalChanges = yield* localChangesReason(worktreePath, settings);
1051
if (finalChanges !== null) return keep(finalChanges);
1012
- const current = resolveWorktreeCleanup(
1013
- yield* settingsService.getSettings,
1014
- thread.projectId,
1015
- );
1052
+ const currentSettings = yield* settingsService.getSettings;
1053
+ const current = resolveWorktreeCleanup(currentSettings, thread.projectId);
1054
if (
1055
+ currentSettings.autoResumeLimitedThreads !== serverSettings.autoResumeLimitedThreads ||
1056
+ currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads ||
1057
Object.keys(settings).some(
1058
(key) =>
1059
current[key as keyof typeof settings] !== settings[key as keyof typeof settings],