The best way to control Claude Code, Codex and more from any computer or phone

main
ts 2,038 lines 87.3 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off
2 import * as NodeFSP from "node:fs/promises";
3 import {
4 CheckpointScopeId,
5 GitCommandError,
6 OrchestrationV2AppThreadJson,
7 OrchestrationV2ProviderSessionJson,
8 } from "@t3tools/contracts";
9 import * as Base64Url from "effect/encoding/Base64Url";
10 import * as Schema from "effect/Schema";
11 import * as SqlClient from "effect/sql/SqlClient";
12 import type {
13 OrchestrationV2ThreadShell,
14 ProjectId,
15 ServerSettings,
16 ServerSettingsError,
17 TerminalSummary,
18 WorktreeCleanupRules,
19 StorageCleanupReport,
20 StorageCleanupReportEntry,
21 ThreadId,
22 } from "@t3tools/contracts";
23 import { resolveWorktreeCleanup } from "@t3tools/shared/projectSettings";
24 import { resolveWorktreeT3Home } from "@t3tools/shared/devHome";
25 import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
26 import * as Context from "effect/Context";
27 import * as Deferred from "effect/Deferred";
28 import * as Layer from "effect/Layer";
29 import * as Clock from "effect/Clock";
30 import * as DateTime from "effect/DateTime";
31 import * as Effect from "effect/Effect";
32 import * as Equal from "effect/Equal";
33 import * as Exit from "effect/Exit";
34 import * as FileSystem from "effect/FileSystem";
35 import * as Option from "effect/Option";
36 import * as Path from "effect/Path";
37 import * as Ref from "effect/Ref";
38 import type { PlatformError } from "effect/PlatformError";
39 import * as Schedule from "effect/Schedule";
40 import * as Stream from "effect/Stream";
41 import * as SubscriptionRef from "effect/SubscriptionRef";
42 import { parse as parseYaml } from "yaml";
43
44 import * as ServerConfig from "./config.ts";
45 import * as GitManager from "./git/GitManager.ts";
46 import * as GitWorkflowService from "./git/GitWorkflowService.ts";
47 import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
48 import { scratchFolderRoot } from "./project/ManagedProjectFolders.ts";
49 import * as Orchestrator from "./orchestration-v2/Orchestrator.ts";
50 import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts";
51 import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
52 import { checkpointRefForScopeOrdinal } from "./orchestration-v2/CheckpointService.ts";
53 import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts";
54 import { MAX_OVERDUE_ARM_MS } from "./orchestration-v2/UsageLimitRecoveryWorker.ts";
55 import * as WorktreeBranchSync from "./orchestration-v2/WorktreeBranchSync.ts";
56 import { forkParked } from "./serverActivation.ts";
57 import * as Settings from "./serverSettings.ts";
58 import * as TerminalManager from "./terminal/Manager.ts";
59 import { isFilesystemRoot, managedWorktreesDirectories } from "./worktreesDirectory.ts";
60 import * as GitVcsDriver from "./vcs/GitVcsDriver.ts";
61 import { isNonRepositoryGitStderr } from "./vcs/GitVcsDriverCore.ts";
62 import { isLinkedWorktree, removeLinkedWorktree } from "./vcs/worktreeRemoval.ts";
63 import { withWorkspaceLease } from "./workspace/workspaceLease.ts";
64 import * as HostProcess from "@t3tools/shared/HostProcess";
65 import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
66 import { isNetworkPath } from "@t3tools/shared/path";
67
68 const decodeCleanupThread = Schema.decodeUnknownEffect(
69 Schema.fromJsonString(OrchestrationV2AppThreadJson),
70 );
71 const decodeCleanupSession = Schema.decodeUnknownEffect(
72 Schema.fromJsonString(OrchestrationV2ProviderSessionJson),
73 );
74 // pnpm 11 writes this file as JSON, older versions as YAML.
75 const decodePnpmModulesManifest = Schema.decodeUnknownEffect(
76 Schema.Struct({
77 storeDir: Schema.String,
78 virtualStoreDir: Schema.optional(Schema.String),
79 }),
80 );
81
82 class PnpmModulesManifestParseError extends Schema.TaggedError<PnpmModulesManifestParseError>()(
83 "PnpmModulesManifestParseError",
84 { cause: Schema.Defect() },
85 ) {
86 override get message(): string {
87 return "Could not parse the pnpm modules manifest.";
88 }
89 }
90
91 class WorktreeDirectoryReadError extends Schema.TaggedError<WorktreeDirectoryReadError>()(
92 "WorktreeDirectoryReadError",
93 { cause: Schema.Defect() },
94 ) {
95 override get message(): string {
96 return "Could not read a worktree directory.";
97 }
98 }
99
100 const DAY_MS = 86_400_000;
101 // Give settle scripts time to open terminals that protect the worktree.
102 const SETTLE_SCRIPT_GRACE_MS = 60_000;
103 // Worktree creation can precede its thread row, including review-guide checkouts.
104 const ORPHAN_WORKTREE_GRACE_MS = 60 * 60 * 1_000;
105
106 const sentence = (text: string) => text.charAt(0).toUpperCase() + text.slice(1);
107 const REPORT_ENTRY_LIMIT = 200;
108 const isGitCommandError = Schema.is(GitCommandError);
109 const errorCode = (error: PlatformError) =>
110 error.cause instanceof Error && "code" in error.cause ? error.cause.code : undefined;
111
112 function cleanupFailureReason(error: { readonly message: string }) {
113 return isGitCommandError(error)
114 ? `${error.command}: ${error.detail.split(/\r?\n/)[0]}${error.reason ? ` (${error.reason})` : ""}`
115 : (error.message.split(/\r?\n/)[0] ?? error.message);
116 }
117
118 const worktreeCleanupEnabled = (rules: WorktreeCleanupRules) =>
119 rules.worktreeAfterDays !== null ||
120 rules.worktreeSettledAfterDays !== null ||
121 rules.worktreeOnMerge ||
122 rules.worktreeOnDelete ||
123 rules.worktreeUnchanged;
124
125 function anyWorktreePolicy(
126 settings: ServerSettings,
127 predicate: (rules: WorktreeCleanupRules) => boolean,
128 ): boolean {
129 return (
130 predicate(resolveWorktreeCleanup(settings, null)) ||
131 Object.keys(settings.projectSettingsOverrides).some((projectId) =>
132 predicate(resolveWorktreeCleanup(settings, projectId as ProjectId)),
133 )
134 );
135 }
136
137 function sameProjectWorktreePolicies(left: ServerSettings, right: ServerSettings): boolean {
138 return [
139 ...new Set([
140 ...Object.keys(left.projectSettingsOverrides),
141 ...Object.keys(right.projectSettingsOverrides),
142 ]),
143 ].every((projectId) =>
144 Equal.equals(
145 left.projectSettingsOverrides[projectId as ProjectId]?.worktreeCleanup,
146 right.projectSettingsOverrides[projectId as ProjectId]?.worktreeCleanup,
147 ),
148 );
149 }
150
151 type LimitRecoverySettings = Pick<
152 ServerSettings,
153 "autoResumeLimitedThreads" | "snoozeLimitedThreads"
154 >;
155
156 export function storageCleanupThreadIdle(
157 thread: OrchestrationV2ThreadShell,
158 now: number,
159 settings: LimitRecoverySettings,
160 ): boolean {
161 return thread.worktreePath !== null && threadIdle(thread, now, settings);
162 }
163
164 function threadIdle(
165 thread: OrchestrationV2ThreadShell,
166 now: number,
167 settings: LimitRecoverySettings,
168 ): boolean {
169 return (
170 thread.activeRunId === null &&
171 (thread.status === "idle" ||
172 thread.status === "completed" ||
173 thread.status === "interrupted" ||
174 thread.status === "failed" ||
175 thread.status === "cancelled" ||
176 thread.status === "rolled_back") &&
177 (thread.pendingBackgroundTasks?.length ?? 0) === 0 &&
178 thread.pendingRuntimeRequest === null &&
179 !threadHasQueuedTurnStart(thread, now) &&
180 !heldForLimitRecovery(thread, now, settings)
181 );
182 }
183
184 // A run held for its usage-limit reset has its work still ahead of it, and a
185 // fresh worktree would otherwise look unchanged and be removed before it runs.
186 function heldForLimitRecovery(
187 thread: OrchestrationV2ThreadShell,
188 now: number,
189 settings: LimitRecoverySettings,
190 ): boolean {
191 if (
192 thread.status !== "failed" ||
193 thread.lastErrorClass !== "usage_limit" ||
194 !thread.usageLimitResetAt ||
195 thread.archivedAt !== null ||
196 thread.settledOverride === "settled"
197 )
198 return false;
199 const resetMs = Date.parse(thread.usageLimitResetAt);
200 const recovery = thread.limitRecovery;
201 if (recovery?.runId === thread.latestRunId && recovery.resetAt === thread.usageLimitResetAt)
202 // An auto-resume holds until its continuation replaces the failed run.
203 return recovery.autoResume || (recovery.snooze === true && resetMs > now);
204 // The recovery worker arms this failure on a later tick; hold it until then.
205 // Like the worker, skip a reset that had already passed when the run ended
206 // or that is too overdue to arm.
207 if (
208 !Number.isFinite(resetMs) ||
209 resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) ||
210 now - resetMs > MAX_OVERDUE_ARM_MS
211 )
212 return false;
213 return settings.autoResumeLimitedThreads || (settings.snoozeLimitedThreads && resetMs > now);
214 }
215
216 /** PR metadata refreshes must not reset the inactivity clock. */
217 export function storageCleanupActivityAt(thread: OrchestrationV2ThreadShell): number {
218 return Math.max(
219 ...[
220 thread.createdAt,
221 thread.latestUserMessageAt,
222 thread.latestRunRequestedAt,
223 thread.latestRunStartedAt,
224 thread.latestRunCompletedAt,
225 ].flatMap((value) => (value == null ? [] : [DateTime.toEpochMillis(value)])),
226 );
227 }
228
229 /** Only a complete subagent or side chat chain within this worktree shares ownership. */
230 function worktreeOwner<T extends Pick<OrchestrationV2ThreadShell, "id" | "lineage">>(
231 threads: ReadonlyArray<T>,
232 ) {
233 if (threads.length === 1) return threads[0];
234 const byId = new Map(threads.map((thread) => [thread.id, thread]));
235 const tops = threads.filter((thread) => !byId.has(thread.lineage.parentThreadId!));
236 if (tops.length !== 1) return undefined;
237 const owner = tops[0]!;
238 for (const thread of threads) {
239 let current = thread;
240 const seen = new Set<string>();
241 while (current.id !== owner.id) {
242 if (
243 seen.has(current.id) ||
244 (current.lineage.relationshipToParent !== "subagent" &&
245 current.lineage.sideChat !== true) ||
246 current.lineage.rootThreadId !== owner.lineage.rootThreadId
247 )
248 return undefined;
249 seen.add(current.id);
250 const parent = byId.get(current.lineage.parentThreadId!);
251 if (parent === undefined) return undefined;
252 current = parent;
253 }
254 }
255 return owner;
256 }
257
258 function worktreeSettlementChain(
259 thread: OrchestrationV2ThreadShell,
260 byId: ReadonlyMap<ThreadId, OrchestrationV2ThreadShell>,
261 now: number,
262 settings: LimitRecoverySettings,
263 ) {
264 const chain = [thread];
265 let current = thread;
266 while (current.settledAt === null) {
267 if (
268 current.lineage.relationshipToParent !== "subagent" ||
269 current.settledOverride === "active" ||
270 current.pinnedAt != null
271 )
272 return [];
273 const parent = byId.get(current.lineage.parentThreadId!);
274 if (
275 parent === undefined ||
276 chain.some((member) => member.id === parent.id) ||
277 parent.projectId !== thread.projectId ||
278 parent.lineage.rootThreadId !== thread.lineage.rootThreadId ||
279 !threadIdle(parent, now, settings)
280 )
281 return [];
282 chain.push(parent);
283 current = parent;
284 }
285 return chain;
286 }
287
288 /** Squash/rebase merges require the merged PR to name this exact head commit. */
289 export function storageCleanupPullRequestMerged(
290 pullRequest: Pick<
291 GitManager.GitBranchPullRequest,
292 "state" | "headRef" | "baseRef" | "headSha"
293 > | null,
294 worktree: {
295 readonly branch: string;
296 readonly defaultBranch: string;
297 readonly headSha: string;
298 readonly integrated: boolean;
299 },
300 ): boolean {
301 return (
302 pullRequest?.state === "merged" &&
303 (worktree.integrated ||
304 (pullRequest.headRef === worktree.branch &&
305 pullRequest.baseRef === worktree.defaultBranch &&
306 pullRequest.headSha === worktree.headSha))
307 );
308 }
309
310 export class StorageCleanup extends Context.Service<
311 StorageCleanup,
312 {
313 readonly runNow: Effect.Effect<StorageCleanupReport, ServerSettingsError>;
314 readonly latestReport: Effect.Effect<StorageCleanupReport | null>;
315 readonly reports: Stream.Stream<StorageCleanupReport | null>;
316 readonly drain: Effect.Effect<void>;
317 }
318 >()("t3/storageCleanup") {}
319
320 const make = Effect.gen(function* () {
321 const config = yield* ServerConfig.ServerConfig;
322 const settingsService = yield* Settings.ServerSettingsService;
323 const projectStore = yield* ProjectStore.ProjectStoreV2;
324 const engine = yield* Orchestrator.OrchestratorV2;
325 const projections = yield* ProjectionStore.ProjectionStoreV2;
326 const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
327 const sql = yield* SqlClient.SqlClient;
328 const git = yield* GitVcsDriver.GitVcsDriver;
329 const gitManager = yield* GitManager.GitManager;
330 const gitWorkflow = yield* GitWorkflowService.GitWorkflowService;
331 const reconcileBranch = yield* WorktreeBranchSync.makeReconcile;
332 const terminals = yield* TerminalManager.TerminalManager;
333 const fs = yield* FileSystem.FileSystem;
334 const path = yield* Path.Path;
335 const liveTerminals = new Map<string, Map<string, TerminalSummary>>();
336 const orphanSizes = new Map<string, number | null>();
337 const noteTerminal = (terminal: TerminalSummary) => {
338 const threadTerminals =
339 liveTerminals.get(terminal.threadId) ?? new Map<string, TerminalSummary>();
340 threadTerminals.set(terminal.terminalId, terminal);
341 liveTerminals.set(terminal.threadId, threadTerminals);
342 };
343
344 const isWindows = yield* HostProcess.isWindows;
345 const pathKey = (target: string) => {
346 const resolved = path.resolve(target);
347 return isWindows ? resolved.toLowerCase() : resolved;
348 };
349 const inside = (root: string, target: string) => {
350 const relative = path.relative(pathKey(root), pathKey(target));
351 return (
352 relative !== "" &&
353 relative !== ".." &&
354 !relative.startsWith(`..${path.sep}`) &&
355 !path.isAbsolute(relative)
356 );
357 };
358 // pnpm imports packages by clone or reflink when its store shares a device
359 // with the worktree. Those files have one link but share blocks with the
360 // store, so removing them frees almost nothing. A store on another device
361 // forces real copies, which still count.
362 const sharedPnpmVirtualStore = Effect.fnUntraced(
363 function* (root: string) {
364 const nodeModules = path.join(root, "node_modules");
365 const text = yield* fs.readFileString(path.join(nodeModules, ".modules.yaml"));
366 const manifest = yield* decodePnpmModulesManifest(
367 yield* Effect.try({
368 try: () => parseYaml(text),
369 catch: (cause) => new PnpmModulesManifestParseError({ cause }),
370 }),
371 );
372 // An agent can write this manifest, and statting a network path makes
373 // Windows sign in to its host with the user's NTLM credentials.
374 if (isNetworkPath(manifest.storeDir) || isNetworkPath(manifest.virtualStoreDir ?? "")) {
375 return null;
376 }
377 const virtualStore = path.resolve(nodeModules, manifest.virtualStoreDir ?? ".pnpm");
378 const [store, virtual] = yield* Effect.all([
379 fs.stat(manifest.storeDir),
380 fs.stat(virtualStore),
381 ]);
382 return store.dev === virtual.dev ? virtualStore : null;
383 },
384 Effect.orElseSucceed(() => null),
385 );
386
387 const measureWorktree = Effect.fn("storageCleanup.measureWorktree")(
388 function* (worktreePath: string, inspectRepositories: boolean | "plain-folder" = false) {
389 const root = yield* fs.realPath(worktreePath);
390 const skipped = yield* sharedPnpmVirtualStore(root);
391 const incomplete = { bytes: null, reason: "worktree could not be fully inspected" };
392 // A plain folder is not a checkout, so a repository at its root counts like one below it.
393 const repositories = new Set(inspectRepositories === "plain-folder" ? [] : [pathKey(root)]);
394 const moduleStores: string[] = [];
395 if (inspectRepositories === true && (yield* fs.exists(path.join(root, ".gitmodules")))) {
396 const submodules = yield* git.execute({
397 operation: "StorageCleanup.nestedRepositories",
398 cwd: root,
399 args: [
400 "submodule",
401 "foreach",
402 "--recursive",
403 "--quiet",
404 'printf "%s\\0" "$toplevel/$sm_path"',
405 ],
406 maxOutputBytes: 64 * 1024,
407 });
408 if (submodules.stdoutTruncated || submodules.stderrTruncated) return incomplete;
409 for (const submodule of submodules.stdout.split("\0").filter(Boolean)) {
410 repositories.add(pathKey(path.resolve(submodule)));
411 repositories.add(pathKey(path.resolve(submodule, ".git")));
412 moduleStores.push(pathKey(path.resolve(submodule, ".git", "modules")));
413 }
414 }
415 // Git status omits ignored repositories, and worktree remove deletes them.
416 // Registered submodules have their own recursive safety checks.
417 const isNestedRepository = (target: string, names: ReadonlyArray<string>) => {
418 const key = pathKey(target);
419 if (
420 repositories.has(key) ||
421 moduleStores.some((store) => key === store || inside(store, key))
422 )
423 return false;
424 const markers = new Set(names.map((name) => name.toLowerCase()));
425 return (
426 markers.has(".git") ||
427 (markers.has("head") && markers.has("objects") && markers.has("refs"))
428 );
429 };
430 const pending = [root];
431 let entries = 1;
432 let bytes = 0;
433 while (pending.length > 0) {
434 const inspected = yield* Effect.forEach(
435 pending.splice(-8),
436 // Untraced: this runs once per entry, and a worktree can hold millions.
437 Effect.fnUntraced(function* (target) {
438 const empty = { target, bytes: 0, names: [], children: [] };
439 const inStore = skipped !== null && (target === skipped || inside(skipped, target));
440 if (inStore && !inspectRepositories) return empty;
441 const stat = yield* fs.lstat(target);
442 if (stat.type === "SymbolicLink") return empty;
443 if ((yield* fs.realPath(target)) !== target) return null;
444 if (stat.type === "Directory") {
445 if (!inStore) {
446 const names = yield* fs.readDirectory(target);
447 return { target, bytes: 0, names, children: names };
448 }
449 // The store is excluded from reclaimed bytes; only walk its directories.
450 const dirents = yield* Effect.tryPromise({
451 try: () => NodeFSP.readdir(target, { withFileTypes: true }),
452 catch: (cause) => new WorktreeDirectoryReadError({ cause }),
453 });
454 return {
455 target,
456 bytes: 0,
457 names: dirents.map((entry) => entry.name),
458 children: dirents.filter((entry) => entry.isDirectory()).map((entry) => entry.name),
459 };
460 }
461 // Removing a shared hardlink leaves its blocks allocated.
462 if (stat.type !== "File" || Option.getOrElse(stat.nlink, () => 1n) !== 1n) return empty;
463 const bytes = Option.match(stat.blocks, {
464 onNone: () => Number(stat.size),
465 onSome: (blocks) => Number(blocks * 512n),
466 });
467 return { ...empty, bytes };
468 }),
469 { concurrency: "unbounded" },
470 );
471 for (const result of inspected) {
472 if (result === null) return incomplete;
473 if (inspectRepositories && isNestedRepository(result.target, result.names))
474 return { bytes: null, reason: "contains a nested Git repository" };
475 bytes += result.bytes;
476 entries += result.names.length;
477 if (entries > 2_000_000) return incomplete;
478 for (const name of result.children) {
479 const child = path.join(result.target, name);
480 if (!inside(root, child)) return incomplete;
481 pending.push(child);
482 }
483 }
484 }
485 return { bytes, reason: null };
486 },
487 Effect.timeout("30 seconds"),
488 Effect.orElseSucceed(() => ({ bytes: null, reason: "worktree could not be fully inspected" })),
489 );
490
491 // Large snapshots take seconds; use the reader worker instead of blocking the event loop.
492 const readThreads = Effect.fn("StorageCleanup.readThreads")(function* () {
493 const snapshot = yield* projections.getShellSnapshot().pipe(NodeSqliteClient.readOnly);
494 const projects = yield* projectStore.listShells();
495 return { projects, threads: [...snapshot.threads, ...snapshot.archivedThreads] };
496 });
497
498 const containsProjectRoot = Effect.fn("StorageCleanup.containsProjectRoot")(function* (
499 worktreePaths: ReadonlyArray<string>,
500 projects: ReadonlyArray<{ readonly workspaceRoot: string }>,
501 ) {
502 const contains = (projectPath: string) =>
503 worktreePaths.some(
504 (worktreePath) =>
505 pathKey(projectPath) === pathKey(worktreePath) || inside(worktreePath, projectPath),
506 );
507 for (const project of projects) {
508 const projectPath = path.resolve(project.workspaceRoot);
509 if (contains(projectPath)) return true;
510 const realPath = yield* fs
511 .realPath(projectPath)
512 .pipe(Effect.orElseSucceed(() => projectPath));
513 if (contains(realPath)) return true;
514 }
515 return false;
516 });
517
518 const reportOrphanWorktrees = Effect.fn("StorageCleanup.reportOrphanWorktrees")(function* (
519 roots: ReadonlyArray<string>,
520 projects: ReadonlyArray<{ readonly workspaceRoot: string }>,
521 entries: StorageCleanupReportEntry[],
522 scratchRoot: string | null,
523 trigger: StorageCleanupReport["trigger"],
524 ) {
525 const rows = yield* sql<{ worktreePath: string }>`
526 SELECT DISTINCT json_extract(payload_json, '$.worktreePath') AS "worktreePath"
527 FROM orchestration_v2_projection_threads
528 WHERE json_extract(payload_json, '$.worktreePath') IS NOT NULL
529 `.pipe(NodeSqliteClient.readOnly);
530 const referenced = new Set<string>();
531 for (const row of rows) {
532 const resolved = path.resolve(row.worktreePath);
533 referenced.add(pathKey(resolved));
534 referenced.add(
535 pathKey(yield* fs.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved))),
536 );
537 }
538 const referencedAncestors = new Set<string>();
539 for (let ancestor of referenced) {
540 while (roots.some((root) => inside(root, ancestor))) {
541 ancestor = pathKey(path.dirname(ancestor));
542 referencedAncestors.add(ancestor);
543 }
544 }
545 const projectRoots = (yield* Effect.forEach(
546 yield* Effect.filter(projects, (project) =>
547 fs.exists(path.resolve(project.workspaceRoot)).pipe(Effect.orElseSucceed(() => true)),
548 ),
549 (project) => {
550 const resolved = path.resolve(project.workspaceRoot);
551 return fs.realPath(resolved).pipe(
552 Effect.orElseSucceed(() => resolved),
553 Effect.map((realPath) => [resolved, realPath]),
554 );
555 },
556 )).flat();
557 const inProject = (target: string, managedRoot: string) =>
558 projectRoots.some(
559 (root) =>
560 !inside(root, managedRoot) && (pathKey(root) === pathKey(target) || inside(root, target)),
561 );
562 // The scratch root is itself a project, so its folders skip the project check.
563 const directories = Effect.fnUntraced(function* (parent: string, managedRoot: string | null) {
564 const skipped = (target: string) => managedRoot !== null && inProject(target, managedRoot);
565 if (skipped(parent)) return [];
566 const result: string[] = [];
567 for (const name of yield* fs.readDirectory(parent).pipe(Effect.orElseSucceed(() => []))) {
568 const target = path.join(parent, name);
569 if (!inside(parent, target) || skipped(target)) continue;
570 const directory = yield* Effect.gen(function* () {
571 if (pathKey(yield* fs.realPath(target)) !== pathKey(target)) return false;
572 return (yield* fs.stat(target)).type === "Directory";
573 }).pipe(Effect.orElseSucceed(() => false));
574 if (directory) result.push(target);
575 }
576 return result;
577 });
578 const now = yield* Clock.currentTimeMillis;
579 const reported = new Map<string, { path: string; reason: string }>();
580 const report = Effect.fnUntraced(
581 function* (worktreePath: string, scratch = false) {
582 const key = pathKey(worktreePath);
583 if (
584 reported.has(key) ||
585 referenced.has(key) ||
586 referencedAncestors.has(key) ||
587 projectRoots.some((root) => pathKey(root) === key || inside(worktreePath, root))
588 )
589 return;
590 const modified = Option.getOrNull((yield* fs.stat(worktreePath)).mtime);
591 if (modified !== null && modified.getTime() > now - ORPHAN_WORKTREE_GRACE_MS) return;
592 const gitPath = path.join(worktreePath, ".git");
593 const linked = (yield* fs.exists(gitPath)) && (yield* isLinkedWorktree(worktreePath));
594 reported.set(key, {
595 path: worktreePath,
596 reason: linked
597 ? "No thread uses this worktree"
598 : scratch
599 ? "No thread uses this folder"
600 : "No thread uses this folder, and it isn't a git worktree",
601 });
602 },
603 Effect.orElseSucceed(() => undefined),
604 );
605 // Linked worktrees can sit deeper inside a plain folder, like <repository>/replay/<n>. Ask
606 // the projects' Git for them rather than walking folders that may hold large non-git trees.
607 let registered: ReadonlyArray<string> | undefined;
608 const nestedWorktrees = Effect.fnUntraced(function* (folder: string, managedRoot: string) {
609 if (yield* fs.exists(path.join(folder, ".git"))) return [];
610 registered ??= (yield* Effect.forEach(new Set(projectRoots), (root) =>
611 git.listWorktreePaths(root).pipe(Effect.orElseSucceed((): string[] => [])),
612 )).flat();
613 // Locked registrations outlive their directory, whose path may since hold something else.
614 return yield* Effect.filter(
615 registered.filter(
616 (worktree) => inside(folder, worktree) && !inProject(worktree, managedRoot),
617 ),
618 (worktree) =>
619 Effect.gen(function* () {
620 if (pathKey(yield* fs.realPath(worktree)) !== pathKey(worktree)) return false;
621 return yield* isLinkedWorktree(worktree);
622 }).pipe(Effect.orElseSucceed(() => false)),
623 );
624 });
625 for (const root of roots) {
626 // Worktrees can be <root>/<repository>/<branch> or explicitly <root>/<name>.
627 for (const projectDirectory of yield* directories(root, root)) {
628 yield* Effect.gen(function* () {
629 if (referenced.has(pathKey(projectDirectory))) return;
630 const gitPath = path.join(projectDirectory, ".git");
631 if (yield* fs.exists(gitPath)) {
632 if (yield* isLinkedWorktree(projectDirectory)) yield* report(projectDirectory);
633 return;
634 }
635 for (const worktreePath of yield* directories(projectDirectory, root)) {
636 const nested = yield* nestedWorktrees(worktreePath, root);
637 for (const target of nested.length > 0 ? nested : [worktreePath]) yield* report(target);
638 }
639 }).pipe(Effect.orElseSucceed(() => undefined));
640 }
641 }
642 if (scratchRoot !== null)
643 for (const folder of yield* directories(scratchRoot, null)) yield* report(folder, true);
644
645 // Discovery runs git probes, so the measurement budget starts after it.
646 const measurementDeadline = (yield* Clock.currentTimeMillis) + 30_000;
647 // Give the first new folder a full walk so every sweep can make progress.
648 let firstMeasurement = true;
649 const refresh = trigger === "manual" ? [...orphanSizes] : [];
650 for (const [key, { path }] of reported) {
651 if (orphanSizes.has(key)) continue;
652 const remaining = measurementDeadline - (yield* Clock.currentTimeMillis);
653 if (!firstMeasurement && remaining <= 0) break;
654 const measured = yield* measureWorktree(path).pipe(
655 firstMeasurement ? Effect.asSome : Effect.timeoutOption(remaining),
656 );
657 firstMeasurement = false;
658 if (Option.isSome(measured)) orphanSizes.set(key, measured.value.bytes);
659 }
660 for (const [key, bytes] of refresh) {
661 const orphan = reported.get(key);
662 if (orphan === undefined || bytes === null) continue;
663 const remaining = measurementDeadline - (yield* Clock.currentTimeMillis);
664 if (remaining <= 0) break;
665 const measured = yield* measureWorktree(orphan.path).pipe(Effect.timeoutOption(remaining));
666 // A slow or failed refresh moves to the end too, so it cannot starve the next orphan.
667 orphanSizes.delete(key);
668 orphanSizes.set(key, Option.getOrUndefined(measured)?.bytes ?? bytes);
669 }
670 for (const [key, orphan] of reported) {
671 entries.push({
672 ...orphan,
673 kind: "worktree",
674 outcome: "kept",
675 projectPath: null,
676 threadId: null,
677 threadTitle: null,
678 bytes: orphanSizes.get(key) ?? null,
679 files: null,
680 });
681 }
682 for (const key of orphanSizes.keys()) {
683 if (!reported.has(key)) orphanSizes.delete(key);
684 }
685 });
686
687 // Use two small Git reads: statusDetailsLocal also diffs and counts for every worktree.
688 const branchChanged = Effect.fn("StorageCleanup.branchChanged")(function* (
689 cwd: string,
690 branch: string | null,
691 ) {
692 const failure = (detail: string) =>
693 new GitCommandError({
694 operation: "StorageCleanup.branchChanged",
695 command: "git",
696 cwd,
697 detail,
698 });
699 // Return null for missing/non-repository cwd; force untranslated git diagnostics for matching.
700 const read = Effect.fnUntraced(function* (args: string[], allowedExitCode?: number) {
701 const result = yield* git
702 .execute({
703 operation: "StorageCleanup.branchChanged",
704 cwd,
705 args,
706 env: { LC_ALL: "C" },
707 allowNonZeroExit: true,
708 })
709 .pipe(
710 Effect.catch((error) =>
711 Effect.flatMap(fs.exists(cwd), (exists) =>
712 exists ? Effect.fail(error) : Effect.succeed(null),
713 ),
714 ),
715 );
716 if (result === null) return null;
717 if (result.exitCode === 0 || result.exitCode === allowedExitCode) return result;
718 if (isNonRepositoryGitStderr(result.stderr)) return null;
719 return yield* failure(
720 result.stderr.trim().split(/\r?\n/)[0]?.slice(0, 1000) || "command failed",
721 );
722 });
723 const index = yield* read(["rev-parse", "--git-path", "index"]);
724 if (index === null) return true;
725 if (yield* fs.exists(`${path.resolve(cwd, index.stdout.trim())}.lock`))
726 return yield* failure(
727 "Git index is locked. Status will resume when the index lock is removed.",
728 );
729 // Detached HEAD exits 1; full refs avoid shortening when a tag shares the branch name.
730 const head = yield* read(["symbolic-ref", "--quiet", "HEAD"], 1);
731 if (head === null) return true;
732 // Removal keeps branches but drops a detached HEAD, so only commits held elsewhere are safe.
733 if (head.exitCode !== 0) {
734 // A paused rebase, merge or bisect would lose its progress (and any autostash) on removal.
735 const operations = yield* read([
736 "rev-parse",
737 "--git-path",
738 "rebase-merge",
739 "--git-path",
740 "rebase-apply",
741 "--git-path",
742 "MERGE_HEAD",
743 "--git-path",
744 "MERGE_AUTOSTASH",
745 "--git-path",
746 "BISECT_LOG",
747 ]);
748 if (operations === null) return true;
749 for (const state of operations.stdout.trim().split(/\r?\n/)) {
750 if (yield* fs.exists(path.resolve(cwd, state))) return true;
751 }
752 const unreachable = yield* read([
753 "rev-list",
754 "--max-count=1",
755 "HEAD",
756 "--not",
757 "--branches",
758 "--remotes",
759 ]);
760 return unreachable === null || unreachable.stdout.trim() !== "";
761 }
762 const ref = head.stdout.trim();
763 const name = ref.startsWith("refs/heads/") ? ref.slice("refs/heads/".length).trim() : null;
764 return name !== branch;
765 });
766
767 // Nothing watches an idle worktree, so its threads still record the branch it left.
768 // Record the checked-out one so the next sweep can apply the rules to it.
769 const followBranch = Effect.fn("StorageCleanup.followBranch")(
770 function* (worktreePath: string) {
771 const cwd = yield* fs.realPath(worktreePath);
772 yield* reconcileBranch({ cwd, refName: yield* gitWorkflow.currentBranch(cwd) });
773 },
774 (effect, worktreePath) =>
775 effect.pipe(
776 Effect.catchCause((cause) =>
777 Effect.logWarning("Failed to reconcile worktree branch", { cwd: worktreePath, cause }),
778 ),
779 ),
780 );
781
782 // Git status and non-forced worktree removal both skip assume-unchanged and skip-worktree files.
783 const hiddenTrackedChange = Effect.fn("StorageCleanup.hiddenTrackedChange")(function* (
784 cwd: string,
785 ) {
786 const inspect = (args: string[], stdin?: string) =>
787 git.execute({
788 operation: "StorageCleanup.hiddenTrackedChanges",
789 cwd,
790 args,
791 ...(stdin === undefined ? {} : { stdin }),
792 maxOutputBytes: 1024 * 1024,
793 });
794 const sparse = yield* inspect([
795 "config",
796 "--type=bool",
797 "--default=false",
798 "core.sparseCheckout",
799 ]);
800 // Sparse checkouts flag every file outside the cone; Git 2.37+ unflags those it finds on disk.
801 const hiddenTag = sparse.stdout.trim() === "true" ? /^[a-z]$/ : /^(?:S|[a-z])$/;
802 const flagged: Array<{ readonly mode: string; readonly id: string; readonly file: string }> =
803 [];
804 let overflow = false;
805 yield* git.execute({
806 operation: "StorageCleanup.hiddenTrackedChanges",
807 cwd,
808 args: ["-c", "core.quotePath=false", "ls-files", "-s", "-v"],
809 maxOutputBytes: 64 * 1024,
810 appendTruncationMarker: true,
811 keepLineCallbacksAfterTruncation: true,
812 progress: {
813 onStdoutLine: (line) =>
814 Effect.sync(() => {
815 const match = /^(\S) (\d+) ([0-9a-f]+) \d\t(.*)$/s.exec(line);
816 if (match === null || !hiddenTag.test(match[1]!) || match[2] === "160000") return;
817 if (flagged.length < 10_000)
818 flagged.push({ mode: match[2]!, id: match[3]!, file: match[4]! });
819 else overflow = true;
820 }),
821 },
822 });
823 if (overflow) return "hidden tracked files exceed the inspection limit";
824 const reason = (file: string) => `has uncommitted changes hidden from git status (${file})`;
825 // Quoted names (control characters, quotes, backslashes) and names that are not valid UTF-8
826 // decode to text that is not the path on disk.
827 const unreadable = flagged.find(({ file }) => file.startsWith('"') || file.includes("\uFFFD"));
828 if (unreadable !== undefined) return reason(unreadable.file);
829 const onDisk = yield* Effect.forEach(
830 flagged,
831 (entry) => {
832 const file = path.join(cwd, entry.file);
833 return fs.readLink(file).pipe(
834 Effect.map((target) => ({ ...entry, type: "SymbolicLink", target })),
835 Effect.catchIf(
836 (error) => errorCode(error) === "EINVAL",
837 () => fs.stat(file).pipe(Effect.map((info) => ({ ...entry, type: info.type }))),
838 ),
839 Effect.catchIf(
840 (error) => errorCode(error) === "ENOENT" || errorCode(error) === "ENOTDIR",
841 () => Effect.succeed(null),
842 ),
843 );
844 },
845 { concurrency: 16 },
846 );
847 const files: Array<{ readonly id: string; readonly file: string }> = [];
848 // Without core.symlinks a tracked link is a regular file holding its target, compared unfiltered.
849 const linkFiles: typeof files = [];
850 for (const entry of onDisk) {
851 if (entry === null) continue;
852 if (entry.type === "File") (entry.mode === "120000" ? linkFiles : files).push(entry);
853 else if (!("target" in entry) || entry.mode !== "120000") return reason(entry.file);
854 // Hashing a link's path would hash the file it points to.
855 else if (
856 (yield* inspect(["hash-object", "--stdin"], entry.target)).stdout.trim() !== entry.id
857 )
858 return reason(entry.file);
859 }
860 const hashPaths = (entries: ReadonlyArray<{ readonly file: string }>, flags: string[]) =>
861 entries.length === 0
862 ? Effect.succeed([])
863 : inspect(
864 ["hash-object", ...flags, "--stdin-paths"],
865 entries.map(({ file }) => `${file}\n`).join(""),
866 ).pipe(Effect.map((result) => result.stdout.split("\n")));
867 const worktreeIds = yield* hashPaths(files, []);
868 const linkIds = yield* hashPaths(linkFiles, ["--no-filters"]);
869 const changed =
870 files.find(({ id }, i) => worktreeIds[i] !== id) ??
871 linkFiles.find(({ id }, i) => linkIds[i] !== id);
872 return changed === undefined ? null : reason(changed.file);
873 });
874
875 // A worktree can outlive its project moving to another clone. Git removes a
876 // worktree through the repository that registered it, not through the
877 // project's current checkout, so resolve the owning repository first.
878 const worktreeRepository = Effect.fn("StorageCleanup.worktreeRepository")(function* (
879 worktreePath: string,
880 ) {
881 const commonDir = yield* git.execute({
882 operation: "StorageCleanup.worktreeRepository",
883 cwd: worktreePath,
884 args: ["rev-parse", "--git-common-dir"],
885 });
886 const resolved = path.resolve(worktreePath, commonDir.stdout.trim());
887 return path.basename(resolved) === ".git" ? path.dirname(resolved) : resolved;
888 });
889
890 const localChangesReason = Effect.fn("StorageCleanup.localChangesReason")(function* (
891 cwd: string,
892 rules: WorktreeCleanupRules,
893 ) {
894 const status = yield* git.execute({
895 operation: "StorageCleanup.localChanges",
896 cwd,
897 args: [
898 "status",
899 "--porcelain=v1",
900 "-z",
901 "--ignore-submodules=none",
902 rules.worktreeKeepWhen === "tracked-changes"
903 ? "--untracked-files=no"
904 : "--untracked-files=all",
905 ],
906 maxOutputBytes: 64 * 1024,
907 appendTruncationMarker: true,
908 });
909 if (status.stdoutTruncated) return "local changes exceed the inspection limit";
910 const records = status.stdout.split("\0").filter(Boolean);
911 let count = 0;
912 for (let i = 0; i < records.length; i++) {
913 count++;
914 if (/^[RC]|^.[RC]/.test(records[i]!)) i++;
915 }
916 if (count > 0) return `has uncommitted changes (${count} ${count === 1 ? "file" : "files"})`;
917 const hidden = yield* hiddenTrackedChange(cwd);
918 if (hidden !== null) return hidden;
919 if (rules.worktreeKeepWhen !== "any-local-files") return null;
920 const ignored = yield* git.execute({
921 operation: "StorageCleanup.ignoredFiles",
922 cwd,
923 args: ["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "-z"],
924 maxOutputBytes: 64 * 1024,
925 appendTruncationMarker: true,
926 });
927 if (ignored.stdoutTruncated) return "ignored files exceed the inspection limit";
928 const file = ignored.stdout
929 .split("\0")
930 .find((entry) => entry !== "" && !/(^|\/)node_modules\/$/.test(entry));
931 return file === undefined ? null : `has ignored files (${file})`;
932 });
933
934 const submoduleRemoval = Effect.fn("StorageCleanup.submoduleRemoval")(
935 function* (cwd: string, rules: WorktreeCleanupRules) {
936 const inspect = (args: string[], maxOutputBytes = 64 * 1024) =>
937 git.execute({
938 operation: "StorageCleanup.submodules",
939 cwd,
940 args,
941 maxOutputBytes,
942 });
943 const admin = yield* inspect(["rev-parse", "--absolute-git-dir"]);
944 if (admin.stdoutTruncated || admin.stderrTruncated)
945 return { force: false, reason: "submodule inspection exceeded the limit" };
946 const modules = path.join(admin.stdout.replace(/\n$/, ""), "modules");
947 const stored = yield* fs.exists(modules);
948 if (!stored && !(yield* fs.exists(path.join(cwd, ".gitmodules"))))
949 return { force: false, reason: null };
950 const populated = yield* inspect([
951 "submodule",
952 "foreach",
953 "--recursive",
954 "--quiet",
955 'printf "%s\\0" "$toplevel/$sm_path"',
956 ]);
957 if (populated.stdoutTruncated || populated.stderrTruncated)
958 return { force: false, reason: "submodule inspection exceeded the limit" };
959 const paths = populated.stdout.split("\0").filter(Boolean);
960 const pending = [modules];
961 for (const submodule of paths) {
962 const directory = yield* inspect(["-C", submodule, "rev-parse", "--absolute-git-dir"]);
963 if (directory.stdoutTruncated || directory.stderrTruncated)
964 return { force: false, reason: "submodule inspection exceeded the limit" };
965 pending.push(directory.stdout.replace(/\n$/, ""));
966 }
967 const visited = new Set<string>();
968 while (pending.length > 0) {
969 const directory = pending.pop()!;
970 if (!(yield* fs.exists(directory))) continue;
971 const canonical = yield* fs.realPath(directory);
972 if (visited.has(canonical)) continue;
973 visited.add(canonical);
974 if (visited.size > 10_000)
975 return { force: false, reason: "submodule inspection exceeded the limit" };
976 if ((yield* fs.stat(directory)).type !== "Directory")
977 return { force: false, reason: "submodule repository could not be inspected" };
978 if (yield* fs.exists(path.join(directory, "HEAD"))) {
979 const unpushed = yield* inspect([
980 "--git-dir",
981 directory,
982 "--work-tree",
983 cwd,
984 "rev-list",
985 "--max-count=1",
986 "HEAD",
987 // Checkpoint refs hold workspace snapshots, not commits a user could lose.
988 "--exclude=refs/t3/*",
989 "--all",
990 "--reflog",
991 "--not",
992 "--remotes",
993 ]);
994 if (unpushed.stdoutTruncated || unpushed.stderrTruncated)
995 return { force: false, reason: "submodule inspection exceeded the limit" };
996 if (unpushed.stdout.trim() !== "")
997 return { force: false, reason: "submodule has commits not on a remote-tracking ref" };
998 pending.push(path.join(directory, "modules"));
999 } else {
1000 for (const name of yield* fs.readDirectory(directory))
1001 pending.push(path.join(directory, name));
1002 }
1003 }
1004 // Check files after walking stored history to shorten the gap before forced removal.
1005 for (const submodule of paths) {
1006 const changes = yield* localChangesReason(submodule, rules);
1007 if (changes !== null) return { force: false, reason: `submodule ${changes}` };
1008 }
1009 const force = stored || paths.length > 0;
1010 if (force && rules.worktreeKeepWhen !== "tracked-changes") {
1011 const root = yield* fs.realPath(cwd);
1012 // Git status and foreach omit files inside deinitialized gitlink directories.
1013 const populatedPaths = new Set(paths.map(pathKey));
1014 const unchecked: string[] = [];
1015 for (const repository of [root, ...paths]) {
1016 const index = yield* inspect(
1017 ["-C", repository, "ls-files", "--stage", "-z"],
1018 16 * 1024 * 1024,
1019 );
1020 if (index.stdoutTruncated || index.stderrTruncated)
1021 return { force: false, reason: "submodule inspection exceeded the limit" };
1022 for (const entry of index.stdout.split("\0")) {
1023 if (!entry.startsWith("160000 ")) continue;
1024 const submodule = path.resolve(repository, entry.slice(entry.indexOf("\t") + 1));
1025 if (!inside(root, submodule))
1026 return { force: false, reason: "submodules could not be inspected" };
1027 if (!populatedPaths.has(pathKey(submodule)) && (yield* fs.exists(submodule)))
1028 unchecked.push(submodule);
1029 }
1030 }
1031 let inspected = unchecked.length;
1032 while (unchecked.length > 0) {
1033 if (inspected > 10_000)
1034 return { force: false, reason: "submodule inspection exceeded the limit" };
1035 const target = unchecked.pop()!;
1036 if (
1037 (yield* fs.realPath(target)) !== path.resolve(target) ||
1038 (yield* fs.stat(target)).type !== "Directory"
1039 )
1040 return { force: false, reason: "submodule has local files" };
1041 if (path.basename(target) === "node_modules") continue;
1042 const names = yield* fs.readDirectory(target);
1043 inspected += names.length;
1044 for (const name of names) unchecked.push(path.join(target, name));
1045 }
1046 }
1047 return { force, reason: null };
1048 },
1049 Effect.orElseSucceed(() => ({ force: false, reason: "submodules could not be inspected" })),
1050 );
1051
1052 const cleanupGit = Effect.fn("StorageCleanup.cleanupGit")(function* (
1053 cwd: string,
1054 args: string[],
1055 command: string,
1056 ) {
1057 const result = yield* git.execute({
1058 operation: "StorageCleanup.cleanupGit",
1059 cwd,
1060 args,
1061 allowNonZeroExit: true,
1062 timeoutMs: 300_000,
1063 });
1064 if (result.exitCode !== 0)
1065 return yield* new GitCommandError({
1066 operation: "StorageCleanup.cleanupGit",
1067 command,
1068 cwd,
1069 detail: result.stderr.trim().split(/\r?\n/)[0]?.slice(0, 1000) || "command failed",
1070 });
1071 });
1072
1073 const cleanWorktrees = Effect.fn("StorageCleanup.cleanWorktrees")(function* (
1074 serverSettings: ServerSettings,
1075 now: number,
1076 entries: StorageCleanupReportEntry[],
1077 trigger: StorageCleanupReport["trigger"],
1078 ) {
1079 if (!anyWorktreePolicy(serverSettings, worktreeCleanupEnabled))
1080 return { waiting: false, kept: [] };
1081 let waiting = false;
1082 const kept: string[] = [];
1083 const roots: Array<string> = [];
1084 // A git worktree's dev home is seeded from real data, so its threads and any
1085 // copied settings can point at the developer's real worktrees.
1086 const isWorktreeDevHome =
1087 (yield* resolveWorktreeT3Home(config.baseDir)) === path.resolve(config.baseDir);
1088 for (const directory of isWorktreeDevHome
1089 ? [config.worktreesDir]
1090 : managedWorktreesDirectories(
1091 serverSettings,
1092 config.worktreesDir,
1093 path,
1094 yield* HostProcess.HomeDirectory,
1095 )) {
1096 const root = yield* fs.exists(directory).pipe(
1097 Effect.flatMap((exists) => (exists ? fs.realPath(directory) : Effect.succeed(null))),
1098 Effect.orElseSucceed(() => null),
1099 );
1100 if (root !== null && !isFilesystemRoot(root, path)) roots.push(root);
1101 }
1102 const scratchRoot = yield* fs
1103 .realPath(scratchFolderRoot(config.baseDir, path))
1104 .pipe(Effect.orElseSucceed(() => null));
1105 // Nothing can bring a scratch folder's files back, so these hold whatever the retention says.
1106 const scratchKeepReason = Effect.fn("StorageCleanup.scratchKeepReason")(function* (
1107 folder: string,
1108 threadIds: ReadonlyArray<string>,
1109 rules: WorktreeCleanupRules,
1110 ) {
1111 const realParent = yield* fs.realPath(path.dirname(folder));
1112 if ((yield* fs.realPath(folder)) !== path.join(realParent, path.basename(folder)))
1113 return "folder is a symbolic link";
1114 // Every file here is untracked, and only "tracked-changes" lets cleanup delete those.
1115 if (rules.worktreeKeepWhen !== "tracked-changes") {
1116 const file = (yield* fs.readDirectory(folder)).find((name) => name !== "node_modules");
1117 if (file !== undefined) return `has local files (${file})`;
1118 }
1119 // Each run of a scheduled task works on what its earlier runs left in the folder.
1120 const scheduled = yield* sql<{ threadId: string }>`
1121 SELECT thread_id AS "threadId" FROM scheduled_tasks
1122 WHERE enabled = 1 AND thread_id IS NOT NULL
1123 `.pipe(NodeSqliteClient.readOnly);
1124 if (scheduled.some((task) => threadIds.includes(task.threadId)))
1125 return "thread has a scheduled task";
1126 return null;
1127 });
1128 const worktreePathReason = Effect.fn("StorageCleanup.worktreePathReason")(function* (
1129 worktreePath: string,
1130 ) {
1131 // Allow symlinked parents, never a symlinked worktree directory.
1132 const realPath = yield* fs.realPath(worktreePath);
1133 const realParent = yield* fs.realPath(path.dirname(worktreePath));
1134 if (realPath !== path.join(realParent, path.basename(worktreePath)))
1135 return "worktree is a symbolic link";
1136 if (!roots.some((root) => inside(root, realPath)))
1137 return "outside the managed worktree folder";
1138 const linked = yield* isLinkedWorktree(worktreePath).pipe(
1139 Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)),
1140 );
1141 if (!linked) return "not a linked worktree";
1142 return null;
1143 });
1144 const deletedRows = yield* sql<{ payload_json: string; workspaceRoot: string }>`
1145 SELECT t.payload_json, p.workspace_root AS "workspaceRoot"
1146 FROM orchestration_v2_projection_threads t
1147 JOIN projection_projects p ON p.project_id = t.project_id
1148 WHERE t.deleted_at IS NOT NULL
1149 `.pipe(NodeSqliteClient.readOnly);
1150 const deletedThreads = (yield* Effect.forEach(deletedRows, (row) =>
1151 decodeCleanupThread(row.payload_json).pipe(
1152 Effect.map((thread) => ({ ...thread, workspaceRoot: row.workspaceRoot })),
1153 ),
1154 )).filter((thread) => thread.worktreePath !== null && thread.branch !== null);
1155 const snapshot = yield* readThreads();
1156 const threadsById = new Map(snapshot.threads.map((thread) => [thread.id, thread]));
1157 // Failures are cached too, so one broken repository costs one fetch per sweep.
1158 const refreshedDefaultRefs = new Map<string, Map<string, Exit.Exit<void, GitCommandError>>>();
1159 const defaultBranches = new Map<string, string | null>();
1160 // Threads may store one checkout through different symlinked spellings; compare real paths.
1161 // Keys are only compared, never opened, so they take pathKey's Windows case folding.
1162 const canonicalPaths = new Map<string, string>();
1163 const canonical = Effect.fnUntraced(function* (target: string) {
1164 const resolved = path.resolve(target);
1165 const cached = canonicalPaths.get(resolved);
1166 if (cached !== undefined) return cached;
1167 const realPath = yield* fs.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved));
1168 const key = pathKey(realPath);
1169 canonicalPaths.set(resolved, key);
1170 return key;
1171 });
1172 // A cwd in a deleted folder cannot be resolved, so stored spellings are compared too.
1173 const occupies = Effect.fnUntraced(function* (
1174 worktreePaths: ReadonlyArray<string>,
1175 target: string,
1176 ) {
1177 const targets = [path.resolve(target), yield* canonical(target)];
1178 return worktreePaths.some((worktree) =>
1179 targets.some((cwd) => pathKey(cwd) === pathKey(worktree) || inside(worktree, cwd)),
1180 );
1181 });
1182 const hasTerminal = Effect.fnUntraced(function* (worktreePaths: ReadonlyArray<string>) {
1183 for (const terminal of [...liveTerminals.values()].flatMap((entries) => [
1184 ...entries.values(),
1185 ])) {
1186 if (terminal.status !== "starting" && terminal.status !== "running") continue;
1187 if (
1188 (terminal.worktreePath !== null &&
1189 (yield* occupies(worktreePaths, terminal.worktreePath))) ||
1190 (yield* occupies(worktreePaths, terminal.cwd))
1191 )
1192 return true;
1193 }
1194 return false;
1195 });
1196 const liveThreads = snapshot.threads.filter((thread) => thread.worktreePath !== null);
1197 const liveKeys = yield* Effect.forEach(liveThreads, (thread) =>
1198 canonical(thread.worktreePath!),
1199 );
1200 const groups = Map.groupBy(liveThreads, (_thread, index) => liveKeys[index]!);
1201 const deletedKeys = yield* Effect.forEach(deletedThreads, (thread) =>
1202 canonical(thread.worktreePath!),
1203 );
1204 const deletedByPath = Map.groupBy(deletedThreads, (_thread, index) => deletedKeys[index]!);
1205 const candidates = [
1206 ...[...groups].map(
1207 ([worktreeKey, group]) =>
1208 // A deleted thread still owns the worktree its remaining subagents share,
1209 // but only deletion cleanup acts on it; otherwise the live rules apply.
1210 deletedByPath
1211 .get(worktreeKey)
1212 ?.find(
1213 (deleted) =>
1214 resolveWorktreeCleanup(serverSettings, deleted.projectId).worktreeOnDelete &&
1215 worktreeOwner([deleted, ...group]) === deleted,
1216 ) ??
1217 worktreeOwner(group) ??
1218 group[0]!,
1219 ),
1220 ...deletedThreads.filter((_thread, index) => !groups.has(deletedKeys[index]!)),
1221 ];
1222 for (const thread of candidates) {
1223 const settings = resolveWorktreeCleanup(serverSettings, thread.projectId);
1224 if (!worktreeCleanupEnabled(settings)) continue;
1225 const worktreePath = path.resolve(thread.worktreePath!);
1226 const worktreeKey = yield* canonical(worktreePath);
1227 const worktreePaths = [worktreePath, worktreeKey];
1228 const group = groups.get(worktreeKey) ?? [];
1229 // No project threads have no branch; their folders sit directly in the scratch root.
1230 const scratch =
1231 thread.branch === null &&
1232 scratchRoot !== null &&
1233 pathKey(path.dirname(worktreeKey)) === pathKey(scratchRoot);
1234 const activityAt = Math.max(...group.map(storageCleanupActivityAt));
1235 const deleted = "workspaceRoot" in thread;
1236 const settlementChain = deleted
1237 ? []
1238 : worktreeSettlementChain(thread, threadsById, now, serverSettings);
1239 const settledThread = settlementChain.at(-1);
1240 const members = deleted ? [thread, ...group] : group;
1241 const project = deleted
1242 ? { workspaceRoot: thread.workspaceRoot }
1243 : snapshot.projects.find((entry) => entry.id === thread.projectId);
1244 const entry: StorageCleanupReportEntry = {
1245 kind: "worktree",
1246 outcome: "kept",
1247 reason: "",
1248 path: worktreePath,
1249 projectPath: project?.workspaceRoot ?? null,
1250 threadId: thread.id,
1251 threadTitle: thread.title,
1252 bytes: null,
1253 files: null,
1254 };
1255 const keep = (reason: string) => {
1256 if (deleted && settings.worktreeOnDelete) kept.push(worktreePath);
1257 entries.push({ ...entry, reason: sentence(reason) });
1258 };
1259 // Reasons a provider session stop or finished deletion work can clear.
1260 const waitForStop = (reason: string) => {
1261 if (deleted && settings.worktreeOnDelete) waiting = true;
1262 return reason;
1263 };
1264 yield* Effect.gen(function* () {
1265 const exists = yield* fs.exists(worktreePath).pipe(
1266 // A stored path under a regular file cannot exist, but access() fails with ENOTDIR.
1267 Effect.catchIf(
1268 (error) =>
1269 error.cause instanceof Error &&
1270 "code" in error.cause &&
1271 error.cause.code === "ENOTDIR",
1272 () => Effect.succeed(false),
1273 ),
1274 );
1275 if (!exists) return;
1276 if (deleted && !settings.worktreeOnDelete) return keep("no rules apply");
1277 if (members.length > 1 && worktreeOwner(members) === undefined)
1278 return keep(`shared by ${members.length} threads`);
1279 if (project === undefined) return keep("project is unavailable");
1280 if (!group.every((member) => storageCleanupThreadIdle(member, now, serverSettings)))
1281 return keep(waitForStop("thread is running or has pending work"));
1282 if (thread.branch === null && !scratch) return keep("thread has no branch");
1283 if (group.some((member) => member.branch !== thread.branch))
1284 return keep("a subagent thread's branch differs from this worktree's branch");
1285 if (yield* hasTerminal(worktreePaths)) return keep(waitForStop("open terminal"));
1286 if (
1287 [...group, ...settlementChain.slice(1)].some(
1288 (member) =>
1289 member.settledAt !== null &&
1290 now - Math.max(DateTime.toEpochMillis(member.updatedAt), activityAt) <
1291 SETTLE_SCRIPT_GRACE_MS,
1292 )
1293 )
1294 return keep(waitForStop("settled moments ago"));
1295 const old =
1296 !deleted &&
1297 settings.worktreeAfterDays !== null &&
1298 activityAt < now - settings.worktreeAfterDays * DAY_MS;
1299 const settledFor =
1300 settledThread?.settledAt == null
1301 ? null
1302 : now - Math.max(DateTime.toEpochMillis(settledThread.settledAt), activityAt);
1303 const settledDays = settledFor === null ? null : Math.floor(settledFor / DAY_MS);
1304 const settled =
1305 settledFor !== null &&
1306 settings.worktreeSettledAfterDays !== null &&
1307 settledFor >= settings.worktreeSettledAfterDays * DAY_MS;
1308 // A worktree is unchanged before its first commit, so only finished threads qualify.
1309 const unchangedApplies =
1310 settings.worktreeUnchanged && (settledThread !== undefined || thread.archivedAt !== null);
1311 let eligible = deleted || old || settled;
1312 let removalReason = deleted
1313 ? "thread was deleted"
1314 : old
1315 ? `inactive for ${Math.floor((now - activityAt) / DAY_MS)} days`
1316 : `settled for ${settledDays} days`;
1317 const notDueReason =
1318 !deleted && settings.worktreeAfterDays !== null
1319 ? `inactive for ${Math.floor((now - activityAt) / DAY_MS)} of ${settings.worktreeAfterDays} days`
1320 : settings.worktreeSettledAfterDays !== null
1321 ? settledFor === null
1322 ? "not settled"
1323 : `settled for ${settledDays} of ${settings.worktreeSettledAfterDays} days`
1324 : settings.worktreeOnMerge
1325 ? "not merged"
1326 : unchangedApplies
1327 ? "has commits beyond the default branch"
1328 : settings.worktreeUnchanged
1329 ? "not settled"
1330 : "no rules apply";
1331 // The merge and unchanged rules read Git, which a scratch folder does not have.
1332 if (scratch && !eligible)
1333 return keep(
1334 settings.worktreeAfterDays !== null || settings.worktreeSettledAfterDays !== null
1335 ? notDueReason
1336 : "no rules apply",
1337 );
1338 // Only Git-based rules can make a not-yet-due worktree eligible, so skip Git otherwise.
1339 if (!eligible && !unchangedApplies && !settings.worktreeOnMerge) return keep(notDueReason);
1340 // Re-read this path's threads and sessions so new activity cancels removal; full per-worktree snapshots take seconds.
1341 const blockedSinceCheck = Effect.fnUntraced(function* () {
1342 const latestProjects = yield* projectStore.listShells();
1343 if (yield* containsProjectRoot(worktreePaths, [project, ...latestProjects]))
1344 return "contains a project checkout";
1345 const bindings = yield* projections.getThreadWorktreePaths({ includeArchived: true });
1346 const bindingKeys = yield* Effect.forEach(bindings, (binding) =>
1347 canonical(binding.worktreePath),
1348 );
1349 const latest = bindings.filter((_binding, index) => bindingKeys[index] === worktreeKey);
1350 if (yield* hasTerminal(worktreePaths)) return waitForStop("open terminal");
1351 if (deleted) {
1352 if (
1353 !resolveWorktreeCleanup(yield* settingsService.getSettings, thread.projectId)
1354 .worktreeOnDelete
1355 )
1356 return "thread or cleanup settings changed since check";
1357 // Durable deletion cleanup must succeed or be explicitly cancelled before removing its checkout.
1358 const pendingCleanup = yield* sql<{ status: string }>`
1359 SELECT status FROM orchestration_v2_effect_outbox
1360 WHERE thread_id = ${thread.id} AND status NOT IN ('succeeded', 'cancelled')
1361 `.pipe(NodeSqliteClient.readOnly);
1362 // Failed effects never retry, so only running work can finish and free the path.
1363 if (pendingCleanup.some((row) => row.status !== "failed"))
1364 return waitForStop("thread deletion is still pending");
1365 if (pendingCleanup.length > 0) return "thread deletion is still pending";
1366 }
1367 const latestShells = yield* Effect.forEach(latest, (entry) =>
1368 projections.getThreadShell(entry.threadId),
1369 );
1370 const latestGroup = latestShells.filter((shell) => shell !== null);
1371 const latestKeys = yield* Effect.forEach(latestGroup, (shell) =>
1372 shell.worktreePath === null ? Effect.succeed(null) : canonical(shell.worktreePath),
1373 );
1374 const latestOwner = worktreeOwner(deleted ? [thread, ...latestGroup] : latestGroup);
1375 if (
1376 latestGroup.length !== group.length ||
1377 latestOwner?.id !== thread.id ||
1378 latestOwner?.branch !== thread.branch ||
1379 latestGroup.some((shell, index) => {
1380 const previous = group.find((member) => member.id === shell.id);
1381 return (
1382 previous === undefined ||
1383 shell.branch !== thread.branch ||
1384 !Equal.equals(shell.settledAt, previous.settledAt) ||
1385 (shell.settledAt !== null && !Equal.equals(shell.updatedAt, previous.updatedAt)) ||
1386 !storageCleanupThreadIdle(shell, now, serverSettings) ||
1387 latestKeys[index] !== worktreeKey ||
1388 storageCleanupActivityAt(shell) !== storageCleanupActivityAt(previous)
1389 );
1390 })
1391 )
1392 return waitForStop("thread activity or shared worktree changed since check");
1393 if (settlementChain.length > 1) {
1394 const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) =>
1395 projections.getThreadShell(ancestor.id),
1396 );
1397 const latestSettlement = worktreeSettlementChain(
1398 latestGroup.find((shell) => shell.id === thread.id)!,
1399 new Map(
1400 [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map(
1401 (ancestor) => [ancestor.id, ancestor],
1402 ),
1403 ),
1404 now,
1405 serverSettings,
1406 ).at(-1);
1407 if (
1408 latestSettlement?.id !== settledThread?.id ||
1409 !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) ||
1410 !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt)
1411 )
1412 return "parent settlement changed since check";
1413 }
1414 // Sessions can outlive their run and can be shared across app threads.
1415 const sessionRows = yield* sql<{ payload_json: string }>`
1416 SELECT payload_json FROM orchestration_v2_projection_provider_sessions
1417 WHERE status != 'stopped'
1418 `.pipe(NodeSqliteClient.readOnly);
1419 const sessions = yield* Effect.forEach(sessionRows, (row) =>
1420 decodeCleanupSession(row.payload_json),
1421 );
1422 // Ready persisted sessions may have released their runtime; only live sessions protect the path.
1423 for (const session of sessions) {
1424 if (
1425 (yield* occupies(worktreePaths, session.cwd)) &&
1426 (yield* providerSessions.isLive(session.id))
1427 )
1428 return waitForStop("provider session is still open");
1429 }
1430 return null;
1431 });
1432 const settingsChangedSinceCheck = Effect.gen(function* () {
1433 const currentSettings = yield* settingsService.getSettings;
1434 const current = resolveWorktreeCleanup(currentSettings, thread.projectId);
1435 return (
1436 currentSettings.autoResumeLimitedThreads !== serverSettings.autoResumeLimitedThreads ||
1437 currentSettings.snoozeLimitedThreads !== serverSettings.snoozeLimitedThreads ||
1438 Object.keys(settings).some(
1439 (key) =>
1440 current[key as keyof typeof settings] !== settings[key as keyof typeof settings],
1441 )
1442 );
1443 });
1444 if (scratch) {
1445 const threadIds = group.map((member) => member.id);
1446 const blocked =
1447 (yield* scratchKeepReason(worktreePath, threadIds, settings)) ??
1448 (yield* blockedSinceCheck());
1449 if (blocked !== null) return keep(blocked);
1450 const inspection = yield* measureWorktree(worktreePath, "plain-folder");
1451 if (inspection.reason !== null) return keep(inspection.reason);
1452 return yield* withWorkspaceLease(
1453 worktreePath,
1454 Effect.gen(function* () {
1455 // Walked again under the lease: a repository deleted with the folder is gone for good.
1456 const finalInspection = yield* measureWorktree(worktreePath, "plain-folder");
1457 if (finalInspection.reason !== null) return keep(finalInspection.reason);
1458 // The walk takes time, so the cheap checks run last, right before removal.
1459 const blockedWhileMeasuring =
1460 (yield* scratchKeepReason(worktreePath, threadIds, settings)) ??
1461 (yield* blockedSinceCheck());
1462 if (blockedWhileMeasuring !== null) return keep(blockedWhileMeasuring);
1463 if (yield* settingsChangedSinceCheck) return keep("settings changed since check");
1464 yield* fs.remove(worktreePath, { recursive: true });
1465 entries.push({
1466 ...entry,
1467 outcome: "removed",
1468 reason: sentence(removalReason),
1469 bytes: inspection.bytes,
1470 });
1471 // The stored path stays so the next turn recreates the folder, empty.
1472 yield* Effect.logInfo("storage cleanup removed scratch folder", {
1473 threadId: thread.id,
1474 });
1475 }),
1476 );
1477 }
1478 const pathReason = yield* worktreePathReason(worktreePath);
1479 if (pathReason !== null) return keep(pathReason);
1480 if (yield* containsProjectRoot(worktreePaths, [project, ...snapshot.projects]))
1481 return keep("contains a project checkout");
1482 if (yield* branchChanged(worktreePath, thread.branch)) {
1483 // A deleted owner cannot follow, so its remaining subagents stay on its branch.
1484 if (!deleted) yield* followBranch(worktreePath);
1485 return keep("repository or branch changed");
1486 }
1487 const changes = yield* localChangesReason(worktreePath, settings);
1488 if (changes !== null) return keep(changes);
1489 const head = yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" });
1490 if (!eligible) {
1491 const repositoryCwd = path.resolve(project.workspaceRoot);
1492 const remotes = yield* git.execute({
1493 operation: "StorageCleanup.listRemotes",
1494 cwd: repositoryCwd,
1495 args: ["remote"],
1496 });
1497 const remote =
1498 remotes.stdout.trim() === ""
1499 ? null
1500 : yield* git.resolvePrimaryRemoteName(repositoryCwd);
1501 let branch: string | null | undefined;
1502 let defaultRef: string;
1503 if (remote === null) {
1504 if (!unchangedApplies)
1505 return keep(
1506 settings.worktreeUnchanged
1507 ? notDueReason
1508 : "no remote is configured for merged-worktree cleanup",
1509 );
1510 const projectHead = yield* git.execute({
1511 operation: "StorageCleanup.resolveProjectBranch",
1512 cwd: repositoryCwd,
1513 args: ["symbolic-ref", "--quiet", "HEAD"],
1514 allowNonZeroExit: true,
1515 });
1516 if (projectHead.exitCode !== 0) return keep("project branch is unavailable");
1517 defaultRef = projectHead.stdout.trim();
1518 branch = defaultRef.slice("refs/heads/".length);
1519 } else {
1520 const defaultBranchKey = `${repositoryCwd}\0${remote}`;
1521 branch = defaultBranches.get(defaultBranchKey);
1522 if (branch === undefined) {
1523 branch = yield* git.resolveDefaultBranchName(repositoryCwd, remote);
1524 // Without a local HEAD ref, only the remote can identify its default;
1525 // a conventional name may be a feature branch.
1526 if (branch === null) {
1527 branch = yield* git
1528 .execute({
1529 operation: "StorageCleanup.resolveDefaultBranch",
1530 cwd: repositoryCwd,
1531 args: ["ls-remote", "--symref", "--", remote, "HEAD"],
1532 timeoutMs: 20_000,
1533 })
1534 .pipe(
1535 Effect.map(
1536 (result) =>
1537 /^ref: refs\/heads\/(.+)\tHEAD\r?$/m.exec(result.stdout)?.[1] ?? null,
1538 ),
1539 Effect.orElseSucceed(() => null),
1540 );
1541 }
1542 defaultBranches.set(defaultBranchKey, branch);
1543 }
1544 if (branch === null) return keep("default branch is unavailable");
1545 defaultRef = `refs/remotes/${remote}/${branch}`;
1546 const refreshed =
1547 refreshedDefaultRefs.get(repositoryCwd) ??
1548 new Map<string, Exit.Exit<void, GitCommandError>>();
1549 let refresh = refreshed.get(defaultRef);
1550 if (refresh === undefined) {
1551 refresh = yield* Effect.exit(
1552 git.fetchRemoteTrackingBranch({
1553 cwd: repositoryCwd,
1554 remoteName: remote,
1555 remoteBranch: branch,
1556 }),
1557 );
1558 refreshed.set(defaultRef, refresh);
1559 refreshedDefaultRefs.set(repositoryCwd, refreshed);
1560 }
1561 yield* refresh;
1562 }
1563 const base = yield* git.resolveCommit({
1564 cwd: worktreePath,
1565 revision: defaultRef,
1566 });
1567 const ancestor = yield* git.execute({
1568 operation: "StorageCleanup.integratedBranch",
1569 cwd: worktreePath,
1570 args: ["merge-base", "--is-ancestor", head.commitSha, base.commitSha],
1571 allowNonZeroExit: true,
1572 });
1573 const integrated = ancestor.exitCode === 0;
1574 eligible = integrated && unchangedApplies;
1575 if (eligible)
1576 removalReason =
1577 remote === null
1578 ? "no commits beyond the project branch"
1579 : "no commits beyond the default branch";
1580 if (!eligible && remote !== null && settings.worktreeOnMerge && thread.branch !== null) {
1581 const pullRequest = yield* gitManager.branchPullRequest(
1582 { cwd: worktreePath, branch: thread.branch },
1583 { refresh: true },
1584 );
1585 eligible = storageCleanupPullRequestMerged(pullRequest, {
1586 branch: thread.branch,
1587 defaultBranch: branch,
1588 headSha: head.commitSha,
1589 integrated,
1590 });
1591 if (eligible) removalReason = "pull request was merged";
1592 }
1593 if (!eligible && remote === null)
1594 return keep(integrated ? notDueReason : "has commits beyond the project branch");
1595 }
1596 if (!eligible) return keep(notDueReason);
1597 const blocked = yield* blockedSinceCheck();
1598 if (blocked !== null) return keep(blocked);
1599 // Inspect before the lease; provider startup must not wait for directory traversal.
1600 const inspection = yield* measureWorktree(worktreePath, true);
1601 if (inspection.reason !== null) return keep(inspection.reason);
1602 const bytes = inspection.bytes;
1603 yield* withWorkspaceLease(
1604 worktreePath,
1605 Effect.gen(function* () {
1606 const pathReason = yield* worktreePathReason(worktreePath);
1607 if (pathReason !== null) return keep(pathReason);
1608 const blockedWhileMeasuring = yield* blockedSinceCheck();
1609 if (blockedWhileMeasuring !== null) return keep(blockedWhileMeasuring);
1610 if (yield* branchChanged(worktreePath, thread.branch))
1611 return keep("repository or branch changed since check");
1612 if (
1613 (yield* git.resolveCommit({ cwd: worktreePath, revision: "HEAD" })).commitSha !==
1614 head.commitSha
1615 )
1616 return keep("commit changed since check");
1617 const finalChanges = yield* localChangesReason(worktreePath, settings);
1618 if (finalChanges !== null) return keep(finalChanges);
1619 if (yield* settingsChangedSinceCheck) return keep("settings changed since check");
1620 const submodules = yield* submoduleRemoval(worktreePath, settings);
1621 if (submodules.reason !== null) return keep(submodules.reason);
1622 if (submodules.force) {
1623 const changes = yield* localChangesReason(worktreePath, settings);
1624 if (changes !== null) return keep(changes);
1625 }
1626 const repository = yield* worktreeRepository(worktreePath);
1627 // Forced removal already discards local files; do not widen its final-check window.
1628 if (settings.worktreeKeepWhen === "tracked-changes" && !submodules.force)
1629 yield* cleanupGit(worktreePath, ["clean", "-ffdx"], "git clean");
1630 yield* removeLinkedWorktree({
1631 operation: "StorageCleanup.cleanupGit",
1632 cwd: repository,
1633 worktreePath,
1634 // Git requires force for submodules; recursive checks above cannot make removal atomic.
1635 force: submodules.force,
1636 execute: git.execute,
1637 run: (args) => cleanupGit(repository, [...args], "git worktree remove"),
1638 });
1639 entries.push({ ...entry, outcome: "removed", reason: sentence(removalReason), bytes });
1640 yield* gitManager.invalidateStatus(project.workspaceRoot);
1641 // Preserve branch/path so ProviderTurnStartService can recreate the checkout on resume.
1642 yield* Effect.logInfo("storage cleanup removed worktree", { threadId: thread.id });
1643 }),
1644 );
1645 }).pipe(
1646 Effect.catch((error) =>
1647 Effect.gen(function* () {
1648 entries.push({
1649 ...entry,
1650 outcome: "failed",
1651 reason: cleanupFailureReason(error),
1652 });
1653 if (deleted && settings.worktreeOnDelete) kept.push(worktreePath);
1654 yield* Effect.logWarning("storage cleanup failed for worktree", {
1655 threadId: thread.id,
1656 error,
1657 });
1658 }),
1659 ),
1660 );
1661 }
1662 yield* reportOrphanWorktrees(
1663 roots,
1664 yield* projectStore.list({ includeDeleted: true }),
1665 entries,
1666 scratchRoot,
1667 trigger,
1668 );
1669 return { waiting, kept };
1670 });
1671
1672 const cleanFiles = Effect.fn("StorageCleanup.cleanFiles")(function* (
1673 root: string,
1674 days: number | null,
1675 now: number,
1676 rotatedLogs: boolean,
1677 removed: { files: number; bytes: number },
1678 ) {
1679 if (days === null || !(yield* fs.exists(root))) return removed;
1680 // A symlinked parent (a linked T3 home) is fine; a symlinked folder is not.
1681 const realRoot = yield* fs.realPath(root);
1682 const realParent = yield* fs.realPath(path.dirname(root));
1683 if (realRoot !== path.join(realParent, path.basename(root))) return removed;
1684 const visit = Effect.fn("StorageCleanup.visitFiles")(function* (
1685 directory: string,
1686 ): Effect.fn.Return<void, PlatformError | ServerSettingsError> {
1687 for (const name of yield* fs.readDirectory(directory)) {
1688 const target = path.join(directory, name);
1689 if ((yield* fs.realPath(target)) !== target || !inside(realRoot, target)) continue;
1690 const stat = yield* fs.stat(target);
1691 if (stat.type === "Directory" && rotatedLogs) {
1692 yield* visit(target);
1693 } else if (stat.type === "File" && (!rotatedLogs || /\.(?:log|ndjson)\.\d+$/.test(name))) {
1694 const modified = Option.getOrNull(stat.mtime);
1695 if (modified !== null && modified.getTime() < now - days * DAY_MS) {
1696 const current = (yield* settingsService.getSettings).storageCleanup;
1697 if ((rotatedLogs ? current.logsAfterDays : current.browserArtifactsAfterDays) !== days)
1698 return;
1699 yield* fs.remove(target);
1700 removed.files++;
1701 removed.bytes += Number(stat.size);
1702 }
1703 }
1704 }
1705 });
1706 yield* visit(realRoot);
1707 return removed;
1708 });
1709
1710 const reportRef = yield* SubscriptionRef.make<StorageCleanupReport | null>(null);
1711 const cleanCheckpointRefs = Effect.fn("StorageCleanup.cleanCheckpointRefs")(function* () {
1712 // Forks display their source history even after its thread is deleted. Keep
1713 // that ancestry, including deleted intermediate forks and archived children.
1714 const rows = yield* sql<{
1715 threadId: string;
1716 worktreePath: string | null;
1717 workspaceRoot: string | null;
1718 scopeId: string | null;
1719 cwd: string | null;
1720 }>`
1721 WITH RECURSIVE retained_checkpoint_threads(thread_id) AS (
1722 SELECT thread_id FROM orchestration_v2_projection_threads WHERE deleted_at IS NULL
1723 UNION
1724 SELECT json_extract(t.payload_json, '$.forkedFrom.threadId')
1725 FROM orchestration_v2_projection_threads t
1726 JOIN retained_checkpoint_threads retained ON retained.thread_id = t.thread_id
1727 WHERE json_extract(t.payload_json, '$.forkedFrom.type') = 'run'
1728 )
1729 SELECT t.thread_id AS "threadId",
1730 json_extract(t.payload_json, '$.worktreePath') AS "worktreePath",
1731 p.workspace_root AS "workspaceRoot", s.scope_id AS "scopeId",
1732 json_extract(s.payload_json, '$.cwd') AS cwd
1733 FROM orchestration_v2_projection_threads t
1734 LEFT JOIN projection_projects p ON p.project_id = t.project_id
1735 LEFT JOIN orchestration_v2_projection_checkpoint_scopes s ON s.thread_id = t.thread_id
1736 WHERE t.deleted_at IS NOT NULL
1737 AND NOT EXISTS (SELECT 1 FROM retained_checkpoint_threads WHERE thread_id = t.thread_id)
1738 AND NOT EXISTS (
1739 SELECT 1 FROM orchestration_v2_effect_outbox e
1740 WHERE e.thread_id = t.thread_id AND e.status IN ('pending', 'running')
1741 )
1742 `.pipe(NodeSqliteClient.readOnly);
1743 const targets = new Map<string, Set<string>>();
1744 for (const row of rows) {
1745 // Derive ownership from tombstones, never from absence in this database:
1746 // another T3 home can own refs in the same Git repository.
1747 const prefixes = [`refs/t3/checkpoints/${Base64Url.encode(row.threadId)}/turn/`];
1748 if (row.scopeId !== null) {
1749 prefixes.push(
1750 (yield* checkpointRefForScopeOrdinal({
1751 scopeId: CheckpointScopeId.make(row.scopeId),
1752 ordinalWithinScope: 0,
1753 })).slice(0, -1),
1754 );
1755 }
1756 // A worktree may already be gone; its refs live in the shared repository.
1757 for (const cwd of [row.cwd, row.worktreePath, row.workspaceRoot]) {
1758 if (cwd === null) continue;
1759 const key = path.resolve(cwd);
1760 const owned = targets.get(key) ?? new Set<string>();
1761 for (const prefix of prefixes) owned.add(prefix);
1762 targets.set(key, owned);
1763 }
1764 }
1765 const repositories = new Map<string, { cwd: string; prefixes: Set<string> }>();
1766 for (const [cwd, prefixes] of targets) {
1767 yield* Effect.gen(function* () {
1768 if (!(yield* fs.exists(cwd))) return;
1769 const result = yield* git.execute({
1770 operation: "StorageCleanup.checkpointRepository",
1771 cwd,
1772 args: ["rev-parse", "--path-format=absolute", "--git-common-dir"],
1773 allowNonZeroExit: true,
1774 });
1775 if (result.exitCode !== 0) return;
1776 const commonDir = yield* fs.realPath(result.stdout.trim());
1777 const repository = repositories.get(commonDir) ?? { cwd, prefixes: new Set<string>() };
1778 for (const prefix of prefixes) repository.prefixes.add(prefix);
1779 repositories.set(commonDir, repository);
1780 }).pipe(
1781 Effect.catch((cause) =>
1782 Effect.logWarning("checkpoint repository inspection failed", { cause }),
1783 ),
1784 );
1785 }
1786 for (const { cwd, prefixes } of repositories.values()) {
1787 yield* Effect.gen(function* () {
1788 const result = yield* git.execute({
1789 operation: "StorageCleanup.listCheckpointRefs",
1790 cwd,
1791 args: [
1792 "for-each-ref",
1793 "--format=%(refname)",
1794 "refs/t3/checkpoints/",
1795 "refs/t3/orchestration-v2/checkpoints/",
1796 ],
1797 maxOutputBytes: 16 * 1024 * 1024,
1798 });
1799 const refs = result.stdout.split("\n").filter((ref) => {
1800 const ordinalStart = ref.lastIndexOf("/") + 1;
1801 return (
1802 prefixes.has(ref.slice(0, ordinalStart)) &&
1803 /^(0|[1-9]\d*)$/.test(ref.slice(ordinalStart))
1804 );
1805 });
1806 if (refs.length === 0) return;
1807 yield* git.execute({
1808 operation: "StorageCleanup.deleteCheckpointRefs",
1809 cwd,
1810 // Remove a symbolic checkpoint itself, never its branch/tag target.
1811 args: ["update-ref", "--no-deref", "-z", "--stdin"],
1812 stdin: refs.map((ref) => `delete ${ref}\0\0`).join(""),
1813 });
1814 yield* Effect.logInfo("storage cleanup removed checkpoint refs", { refCount: refs.length });
1815 }).pipe(
1816 Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })),
1817 );
1818 }
1819 });
1820 // Only deleted threads' worktrees wait on session stops: any stop while one waits, or a stop inside one kept for another reason.
1821 // A deletion stays pending until a sweep that started after it finishes.
1822 let deletedWorktreeWaits = true;
1823 let keptDeletedWorktrees: ReadonlyArray<string> = [];
1824 let deletions = 0;
1825 let sweptDeletions = 0;
1826 const sweep = Effect.fn("StorageCleanup.sweep")(function* (
1827 trigger: StorageCleanupReport["trigger"],
1828 ) {
1829 const deletionsAtStart = deletions;
1830 const now = yield* Clock.currentTimeMillis;
1831 const serverSettings = yield* settingsService.getSettings;
1832 const settings = serverSettings.storageCleanup;
1833 const entries: StorageCleanupReportEntry[] = [];
1834 yield* cleanCheckpointRefs().pipe(
1835 Effect.catch((cause) => Effect.logWarning("checkpoint ref cleanup failed", { cause })),
1836 );
1837 const deletedWorktrees = yield* cleanWorktrees(serverSettings, now, entries, trigger).pipe(
1838 Effect.catch((error) =>
1839 Effect.gen(function* () {
1840 entries.push({
1841 kind: "worktree",
1842 outcome: "failed",
1843 reason: cleanupFailureReason(error),
1844 path: null,
1845 projectPath: null,
1846 threadId: null,
1847 threadTitle: null,
1848 bytes: null,
1849 files: null,
1850 });
1851 yield* Effect.logWarning("worktree cleanup failed", { error });
1852 return { waiting: true, kept: [] };
1853 }),
1854 ),
1855 );
1856 deletedWorktreeWaits = deletedWorktrees.waiting;
1857 keptDeletedWorktrees = deletedWorktrees.kept;
1858 sweptDeletions = deletionsAtStart;
1859 for (const category of [
1860 {
1861 kind: "browser-artifacts" as const,
1862 root: config.browserArtifactsDir,
1863 days: settings.browserArtifactsAfterDays,
1864 label: "browser artifacts",
1865 },
1866 {
1867 kind: "logs" as const,
1868 root: config.logsDir,
1869 days: settings.logsAfterDays,
1870 label: "rotated logs",
1871 },
1872 ]) {
1873 if (category.days === null) continue;
1874 const removed = { files: 0, bytes: 0 };
1875 yield* cleanFiles(category.root, category.days, now, category.kind === "logs", removed).pipe(
1876 Effect.map(({ files, bytes }) =>
1877 entries.push({
1878 kind: category.kind,
1879 outcome: files > 0 ? "removed" : "kept",
1880 reason: files === 0 ? "No expired files" : `Removed ${files} ${category.label}`,
1881 path: null,
1882 projectPath: null,
1883 threadId: null,
1884 threadTitle: null,
1885 bytes: files > 0 ? bytes : null,
1886 files: files > 0 ? files : null,
1887 }),
1888 ),
1889 Effect.catch((error) =>
1890 Effect.gen(function* () {
1891 entries.push({
1892 kind: category.kind,
1893 outcome: "failed",
1894 reason: cleanupFailureReason(error),
1895 path: null,
1896 projectPath: null,
1897 threadId: null,
1898 threadTitle: null,
1899 bytes: removed.files > 0 ? removed.bytes : null,
1900 files: removed.files > 0 ? removed.files : null,
1901 });
1902 yield* Effect.logWarning("storage file cleanup failed", { kind: category.kind, error });
1903 }),
1904 ),
1905 );
1906 }
1907 const counts = { removed: 0, kept: 0, failed: 0 };
1908 let bytesFreed = 0;
1909 for (const entry of entries) {
1910 counts[entry.outcome]++;
1911 if (entry.outcome !== "kept") bytesFreed += entry.bytes ?? 0;
1912 }
1913 const priority = (entry: StorageCleanupReportEntry) =>
1914 entry.outcome === "failed"
1915 ? 0
1916 : entry.outcome === "removed"
1917 ? 1
1918 : entry.kind === "worktree" && entry.threadId === null
1919 ? 2
1920 : 3;
1921 const latestReport: StorageCleanupReport = {
1922 trigger,
1923 startedAt: DateTime.formatIso(DateTime.makeUnsafe(now)),
1924 finishedAt: DateTime.formatIso(yield* DateTime.now),
1925 entries: entries.sort((a, b) => priority(a) - priority(b)).slice(0, REPORT_ENTRY_LIMIT),
1926 counts,
1927 bytesFreed,
1928 omittedCount: Math.max(0, entries.length - REPORT_ENTRY_LIMIT),
1929 };
1930 yield* SubscriptionRef.set(reportRef, latestReport);
1931 return latestReport;
1932 });
1933 const automaticSweepQueued = yield* Ref.make(false);
1934 const worker = yield* makeDrainableWorker(
1935 (completion: Deferred.Deferred<StorageCleanupReport, ServerSettingsError> | undefined) =>
1936 (completion === undefined ? Ref.set(automaticSweepQueued, false) : Effect.void).pipe(
1937 Effect.andThen(sweep(completion === undefined ? "automatic" : "manual")),
1938 Effect.exit,
1939 Effect.flatMap((exit) =>
1940 completion === undefined
1941 ? exit.pipe(
1942 Effect.asVoid,
1943 Effect.catchCause((cause) =>
1944 Effect.logWarning("storage cleanup failed", { cause }),
1945 ),
1946 )
1947 : Deferred.done(completion, exit).pipe(Effect.asVoid),
1948 ),
1949 ),
1950 );
1951 const requestSweep = Ref.getAndSet(automaticSweepQueued, true).pipe(
1952 Effect.flatMap((queued) => (queued ? Effect.void : worker.enqueue(undefined))),
1953 );
1954 const runNow = Effect.gen(function* () {
1955 const completion = yield* Deferred.make<StorageCleanupReport, ServerSettingsError>();
1956 yield* worker.enqueue(completion);
1957 return yield* Deferred.await(completion);
1958 });
1959
1960 const start = Effect.fn("StorageCleanup.start")(function* () {
1961 const unsubscribe = yield* terminals.subscribeMetadata((event) =>
1962 Effect.sync(() => {
1963 if (event.type === "snapshot") {
1964 liveTerminals.clear();
1965 for (const terminal of event.terminals) noteTerminal(terminal);
1966 } else if (event.type === "upsert") {
1967 noteTerminal(event.terminal);
1968 } else {
1969 const threadTerminals = liveTerminals.get(event.threadId);
1970 threadTerminals?.delete(event.terminalId);
1971 if (threadTerminals?.size === 0) liveTerminals.delete(event.threadId);
1972 }
1973 }),
1974 );
1975 yield* Effect.addFinalizer(() => Effect.sync(unsubscribe));
1976 const changes = yield* settingsService.subscribeChanges;
1977 const events = engine.streamDomainEvents;
1978 let lastSettings = yield* settingsService.getSettings.pipe(Effect.orDie);
1979 yield* forkParked(
1980 requestSweep.pipe(
1981 Effect.andThen(worker.drain),
1982 Effect.repeat(Schedule.spaced("1 hour")),
1983 Effect.asVoid,
1984 ),
1985 );
1986 yield* forkParked(
1987 Stream.runForEach(changes, (settings) => {
1988 if (
1989 Equal.equals(settings.storageCleanup, lastSettings.storageCleanup) &&
1990 Equal.equals(settings.worktreeCleanup, lastSettings.worktreeCleanup) &&
1991 sameProjectWorktreePolicies(settings, lastSettings)
1992 )
1993 return Effect.void;
1994 lastSettings = settings;
1995 return requestSweep;
1996 }),
1997 );
1998 yield* forkParked(
1999 Stream.runForEach(events, (event) => {
2000 if (event.type === "thread.deleted") deletions++;
2001 else if (
2002 event.type !== "provider-session.updated" ||
2003 (event.payload.status !== "stopped" && event.payload.status !== "error")
2004 )
2005 return Effect.void;
2006 else if (!deletedWorktreeWaits && deletions === sweptDeletions) {
2007 const cwd = pathKey(event.payload.cwd);
2008 if (
2009 !keptDeletedWorktrees.some(
2010 (worktreePath) => pathKey(worktreePath) === cwd || inside(worktreePath, cwd),
2011 )
2012 )
2013 return Effect.void;
2014 }
2015 if (event.type === "thread.deleted") return requestSweep;
2016 if (!anyWorktreePolicy(lastSettings, (rules) => rules.worktreeOnDelete)) return Effect.void;
2017 return event.type === "provider-session.updated" && event.payload.status === "error"
2018 ? providerSessions
2019 .isLive(event.payload.id)
2020 .pipe(Effect.flatMap((live) => (live ? Effect.void : requestSweep)))
2021 : requestSweep;
2022 }).pipe(
2023 Effect.catchCause((cause) =>
2024 Effect.logWarning("Storage cleanup event stream failed", { cause }),
2025 ),
2026 ),
2027 );
2028 });
2029 yield* start();
2030 return StorageCleanup.of({
2031 runNow,
2032 latestReport: SubscriptionRef.get(reportRef),
2033 reports: SubscriptionRef.changes(reportRef),
2034 drain: worker.drain,
2035 });
2036 });
2037
2038 export const layer = Layer.effect(StorageCleanup, make);