@devries-cameron20 / t3code-1 / commits / d7d5d7c79b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): cleanup keeps worktrees waiting for a usage-limit reset (#24689)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>

Victor Vogel committed Oct 31, 2026 at 03:47 UTC d7d5d7c79b704ea28d1e10f7b9bcc7bd525cd771
2 files changed +151 -13
apps/server/src/storageCleanup.test.ts
+103 -5
@@ -112,14 +112,16 @@ describe("V2 storage cleanup eligibility", () => {
112 const candidate = () => shell({ branch: "feature", worktreePath: "/worktrees/feature" });
113
114 it("allows an idle worktree and rejects the project checkout", () => {
115 - expect(storageCleanupThreadIdle(candidate(), NOW_MS)).toBe(true);
116 - expect(storageCleanupThreadIdle(shell(), NOW_MS)).toBe(false);
115 + expect(storageCleanupThreadIdle(candidate(), NOW_MS, DEFAULT_SERVER_SETTINGS)).toBe(true);
116 + expect(storageCleanupThreadIdle(shell(), NOW_MS, DEFAULT_SERVER_SETTINGS)).toBe(false);
117 });
118
119 it.each(["running", "starting", "preparing", "waiting", "queued"] as const)(
120 "retains a worktree while its thread is %s",
121 (status) => {
122 - expect(storageCleanupThreadIdle(candidateWithStatus(status), NOW_MS)).toBe(false);
122 + expect(
123 + storageCleanupThreadIdle(candidateWithStatus(status), NOW_MS, DEFAULT_SERVER_SETTINGS),
124 + ).toBe(false);
125 },
126 );
127
@@ -133,6 +135,7 @@ describe("V2 storage cleanup eligibility", () => {
135 pendingBackgroundTasks: [{ taskId: "task-1", kind: "command" }],
136 },
137 NOW_MS,
138 + DEFAULT_SERVER_SETTINGS,
139 ),
140 ).toBe(false);
141 },
@@ -152,6 +155,7 @@ describe("V2 storage cleanup eligibility", () => {
155 },
156 },
157 NOW_MS,
158 + DEFAULT_SERVER_SETTINGS,
159 ),
160 ).toBe(false);
161 },
@@ -159,16 +163,83 @@ describe("V2 storage cleanup eligibility", () => {
163
164 it("retains an active run even if the shell status is idle", () => {
165 expect(
162 - storageCleanupThreadIdle({ ...candidate(), activeRunId: RunId.make("run") }, NOW_MS),
166 + storageCleanupThreadIdle(
167 + { ...candidate(), activeRunId: RunId.make("run") },
168 + NOW_MS,
169 + DEFAULT_SERVER_SETTINGS,
170 + ),
171 ).toBe(false);
172 });
173
174 it("retains a queued prompt before the new run has been projected", () => {
175 expect(
168 - storageCleanupThreadIdle({ ...candidate(), latestUserMessageAt: at(-1_000) }, NOW_MS),
176 + storageCleanupThreadIdle(
177 + { ...candidate(), latestUserMessageAt: at(-1_000) },
178 + NOW_MS,
179 + DEFAULT_SERVER_SETTINGS,
180 + ),
181 ).toBe(false);
182 });
183
184 + it.each([
185 + { name: "snoozed until a future reset", snooze: true, resetIn: DAY_MS, idle: false },
186 + { name: "set to auto-resume", autoResume: true, resetIn: DAY_MS, idle: false },
187 + { name: "with recovery turned off", resetIn: DAY_MS, idle: true },
188 + { name: "auto-resuming after the reset", autoResume: true, resetIn: -1_000, idle: false },
189 + { name: "snoozed past its reset", snooze: true, resetIn: -1_000, idle: true },
190 + {
191 + name: "settled with an auto-resume pending",
192 + autoResume: true,
193 + settled: true,
194 + resetIn: -1_000,
195 + idle: true,
196 + },
197 + { name: "not yet armed by the recovery worker", armed: false, resetIn: DAY_MS, idle: false },
198 + {
199 + name: "not yet armed, auto-resume after reset",
200 + armed: false,
201 + autoResumeSetting: true,
202 + resetIn: -1_000,
203 + idle: false,
204 + },
205 + {
206 + name: "not armed with recovery settings off",
207 + armed: false,
208 + settingsOff: true,
209 + resetIn: DAY_MS,
210 + idle: true,
211 + },
212 + ])(
213 + "treats a usage-limited thread $name as idle: $idle",
214 + ({
215 + autoResume = false,
216 + snooze = false,
217 + armed = true,
218 + autoResumeSetting = false,
219 + settled,
220 + settingsOff,
221 + resetIn,
222 + idle,
223 + }) => {
224 + const runId = RunId.make("run-1");
225 + const resetAt = DateTime.formatIso(at(resetIn));
226 + expect(
227 + storageCleanupThreadIdle(
228 + {
229 + ...candidateWithStatus("failed"),
230 + latestRunId: runId,
231 + lastErrorClass: "usage_limit",
232 + usageLimitResetAt: resetAt,
233 + settledOverride: settled ? "settled" : null,
234 + limitRecovery: armed ? { runId, resetAt, autoResume, snooze } : null,
235 + },
236 + NOW_MS,
237 + { autoResumeLimitedThreads: autoResumeSetting, snoozeLimitedThreads: !settingsOff },
238 + ),
239 + ).toBe(idle);
240 + },
241 + );
242 +
243 it("uses V2 run activity instead of metadata refreshes for retention", () => {
244 const thread = candidate();
245 const runTime = at(-3 * DAY_MS);
@@ -2283,6 +2354,33 @@ describe("storage cleanup reports and local file policies", () => {
2354 });
2355
2356 describe("settled worktree rechecks", () => {
2357 + it.live.each(["autoResumeLimitedThreads", "snoozeLimitedThreads"] as const)(
2358 + "keeps a usage-limited checkout when %s is enabled during measurement",
2359 + (preference) =>
2360 + runCleanupTest(
2361 + Effect.gen(function* () {
2362 + const fixture = yield* pausedMeasurementFixture();
2363 + fixture.setSettings({ autoResumeLimitedThreads: false, snoozeLimitedThreads: false });
2364 + fixture.setThreads([
2365 + shell({
2366 + branch: "feature",
2367 + worktreePath: fixture.worktree,
2368 + status: "failed",
2369 + latestRunId: RunId.make("limited-run"),
2370 + lastErrorClass: "usage_limit",
2371 + usageLimitResetAt: DateTime.formatIso(at(DAY_MS)),
2372 + }),
2373 + ]);
2374 + const run = yield* fixture.service.runNow.pipe(Effect.forkChild);
2375 + yield* Deferred.await(fixture.measuring);
2376 + fixture.setSettings({ [preference]: true });
2377 + yield* Deferred.succeed(fixture.resume, undefined);
2378 + const report = yield* Fiber.join(run);
2379 + expect(report.entries[0]?.outcome).toBe("kept");
2380 + expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true);
2381 + }),
2382 + ),
2383 + );
2384 it.live.each([{ settledAt: null }, { settledAt: at(DAY_MS) }])(
2385 "keeps a checkout changed during measurement (%j)",
2386 (change) =>
apps/server/src/storageCleanup.ts
+48 -8
@@ -115,7 +115,16 @@ function sameProjectWorktreePolicies(left: ServerSettings, right: ServerSettings
115 );
116 }
117
118 -export function storageCleanupThreadIdle(thread: OrchestrationV2ThreadShell, now: number): boolean {
118 +type LimitRecoverySettings = Pick<
119 + ServerSettings,
120 + "autoResumeLimitedThreads" | "snoozeLimitedThreads"
121 +>;
122 +
123 +export function storageCleanupThreadIdle(
124 + thread: OrchestrationV2ThreadShell,
125 + now: number,
126 + settings: LimitRecoverySettings,
127 +): boolean {
128 return (
129 thread.worktreePath !== null &&
130 thread.activeRunId === null &&
@@ -127,10 +136,41 @@ export function storageCleanupThreadIdle(thread: OrchestrationV2ThreadShell, now
136 thread.status === "rolled_back") &&
137 (thread.pendingBackgroundTasks?.length ?? 0) === 0 &&
138 thread.pendingRuntimeRequest === null &&
130 - !threadHasQueuedTurnStart(thread, now)
139 + !threadHasQueuedTurnStart(thread, now) &&
140 + !heldForLimitRecovery(thread, now, settings)
141 );
142 }
143
144 +// A run held for its usage-limit reset has its work still ahead of it, and a
145 +// fresh worktree would otherwise look unchanged and be removed before it runs.
146 +function heldForLimitRecovery(
147 + thread: OrchestrationV2ThreadShell,
148 + now: number,
149 + settings: LimitRecoverySettings,
150 +): boolean {
151 + if (
152 + thread.status !== "failed" ||
153 + thread.lastErrorClass !== "usage_limit" ||
154 + !thread.usageLimitResetAt ||
155 + thread.archivedAt !== null ||
156 + thread.settledOverride === "settled"
157 + )
158 + return false;
159 + const resetMs = Date.parse(thread.usageLimitResetAt);
160 + const recovery = thread.limitRecovery;
161 + if (recovery?.runId === thread.latestRunId && recovery.resetAt === thread.usageLimitResetAt)
162 + // An auto-resume holds until its continuation replaces the failed run.
163 + return recovery.autoResume || (recovery.snooze === true && resetMs > now);
164 + // The recovery worker arms this failure on a later tick; hold it until then.
165 + // Like the worker, skip a reset that had already passed when the run ended.
166 + if (
167 + !Number.isFinite(resetMs) ||
168 + resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt)
169 + )
170 + return false;
171 + return settings.autoResumeLimitedThreads || (settings.snoozeLimitedThreads && resetMs > now);
172 +}
173 +
174 /** PR metadata refreshes must not reset the inactivity clock. */
175 export function storageCleanupActivityAt(thread: OrchestrationV2ThreadShell): number {
176 return Math.max(
@@ -800,7 +840,7 @@ const make = Effect.gen(function* () {
840 if (project === undefined) return keep("project is unavailable");
841 if (
842 thread.branch === null ||
803 - !group.every((member) => storageCleanupThreadIdle(member, now))
843 + !group.every((member) => storageCleanupThreadIdle(member, now, serverSettings))
844 )
845 return keep(waitForStop("thread is running or has pending work"));
846 if (group.some((member) => member.branch !== thread.branch))
@@ -969,7 +1009,7 @@ const make = Effect.gen(function* () {
1009 shell.branch !== thread.branch ||
1010 !Equal.equals(shell.settledAt, previous.settledAt) ||
1011 (shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) ||
972 - !storageCleanupThreadIdle(shell, now) ||
1012 + !storageCleanupThreadIdle(shell, now, serverSettings) ||
1013 path.resolve(shell.worktreePath!) !== worktreePath ||
1014 storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous)
1015 );
@@ -1009,11 +1049,11 @@ const make = Effect.gen(function* () {
1049 return keep("commit changed since check");
1050 const finalChanges = yield* localChangesReason(worktreePath, settings);
1051 if (finalChanges !== null) return keep(finalChanges);
1012 - const current = resolveWorktreeCleanup(
1013 - yield* settingsService.getSettings,
1014 - thread.projectId,
1015 - );
1052 + const currentSettings = yield* settingsService.getSettings;
1053 + const current = resolveWorktreeCleanup(currentSettings, thread.projectId);
1054 if (
1055 + currentSettings.autoResumeLimitedThreads !== serverSettings.autoResumeLimitedThreads ||
1056 + currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads ||
1057 Object.keys(settings).some(
1058 (key) =>
1059 current[key as keyof typeof settings] !== settings[key as keyof typeof settings],