The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off - Git index stat data needs bigint lstat, which FileSystem lacks. |
| 2 | import * as NodeCrypto from "node:crypto"; |
| 3 | import * as NodeFSP from "node:fs/promises"; |
| 4 | |
| 5 | import { GitCommandError } from "@t3tools/contracts"; |
| 6 | import * as HostProcess from "@t3tools/shared/HostProcess"; |
| 7 | import * as Clock from "effect/Clock"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Exit from "effect/Exit"; |
| 10 | import * as FileSystem from "effect/FileSystem"; |
| 11 | import * as Option from "effect/Option"; |
| 12 | import * as Path from "effect/Path"; |
| 13 | import * as Schema from "effect/Schema"; |
| 14 | import * as Semaphore from "effect/Semaphore"; |
| 15 | |
| 16 | import * as CheckoutCloner from "./CheckoutCloner.ts"; |
| 17 | import type * as GitVcsDriver from "./GitVcsDriver.ts"; |
| 18 | import { indexEntryStat, parseIndex, restatIndex, type IndexEntryStat } from "./gitIndexStat.ts"; |
| 19 | |
| 20 | const KEEP_PER_REPOSITORY = 3; |
| 21 | // Moving a cached checkout costs about 0.1 ms per changed file when a worktree is created, while |
| 22 | // replacing it means writing and later deleting a whole checkout, which slows the next clone. |
| 23 | const REBUILD_AFTER_CHANGED_FILES = 500; |
| 24 | const UNUSED_REPOSITORY_MS = 30 * 24 * 60 * 60 * 1000; |
| 25 | // Older leftovers came from a crash, not from work in progress. A full build of a huge repository |
| 26 | // can take minutes, and its staging folder's own mtime does not change while files land inside. |
| 27 | const ABANDONED_STAGING_MS = 24 * 60 * 60 * 1000; |
| 28 | const ABANDONED_TRASH_MS = 10 * 60 * 1000; |
| 29 | const STAT_BATCH = 512; |
| 30 | const TREE_ID = /^[0-9a-f]{40}$/; |
| 31 | // `<tree>-<fingerprint>`: checkouts of one tree under different settings never replace each other. |
| 32 | const ENTRY_NAME = /^([0-9a-f]{40})-([0-9a-f]{16})$/; |
| 33 | const GIT_FALSE = new Set(["false", "no", "off", "0"]); |
| 34 | const MAX_LIST_BYTES = 256 * 1024 * 1024; |
| 35 | const NULL_OID = "0".repeat(40); |
| 36 | // 2.42 names its system attributes file, which the fingerprint needs; `git hook run` is from 2.36. |
| 37 | const MIN_GIT = [2, 42] as const; |
| 38 | // Settings that change what a checkout writes. Read in the new worktree, so conditional includes apply. |
| 39 | const CHECKOUT_CONFIG = |
| 40 | "^(core\\.(autocrlf|eol|symlinks|safecrlf|checkroundtripencoding|precomposeunicode|ignorecase|filemode|attributesfile|sparsecheckout)|filter\\.|lfs\\.|extensions\\.)"; |
| 41 | // Keeps cache indexes in the one format the stat rewriter reads. |
| 42 | const CACHE_INDEX_CONFIG = [ |
| 43 | "index.version=2", |
| 44 | "index.skipHash=false", |
| 45 | "core.splitIndex=false", |
| 46 | "core.untrackedCache=false", |
| 47 | "core.fsmonitor=false", |
| 48 | "core.sparseCheckout=false", |
| 49 | // Private builds must not run the repository's hooks. |
| 50 | "core.hooksPath=/dev/null", |
| 51 | "checkout.workers=0", |
| 52 | ].flatMap((setting) => ["-c", setting]); |
| 53 | |
| 54 | // How a case- and normalization-insensitive volume such as default APFS compares names. |
| 55 | const foldCase = (file: string) => file.normalize("NFC").toLowerCase(); |
| 56 | |
| 57 | // `check-attr --all -z` output: any `filter` attribute counts, as `unset` can also name a driver. |
| 58 | function namesFilter(output: string): boolean { |
| 59 | const fields = output.split("\0"); |
| 60 | for (let index = 1; index < fields.length; index += 3) { |
| 61 | if (fields[index] === "filter") return true; |
| 62 | } |
| 63 | return false; |
| 64 | } |
| 65 | |
| 66 | /** Every folder that holds one of `paths`, including the root as "". */ |
| 67 | function parentDirectories(paths: Iterable<string>): Set<string> { |
| 68 | const directories = new Set([""]); |
| 69 | for (const entryPath of paths) { |
| 70 | for (let slash = entryPath.lastIndexOf("/"); slash > 0;) { |
| 71 | const directory = entryPath.slice(0, slash); |
| 72 | if (directories.has(directory)) break; |
| 73 | directories.add(directory); |
| 74 | slash = directory.lastIndexOf("/"); |
| 75 | } |
| 76 | } |
| 77 | return directories; |
| 78 | } |
| 79 | |
| 80 | class WorktreeCheckoutCacheError extends Schema.TaggedError<WorktreeCheckoutCacheError>()( |
| 81 | "WorktreeCheckoutCacheError", |
| 82 | { detail: Schema.String }, |
| 83 | ) { |
| 84 | override get message(): string { |
| 85 | return this.detail; |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | interface CacheEntry { |
| 90 | readonly directory: string; |
| 91 | readonly tree: string; |
| 92 | readonly fingerprint: string; |
| 93 | readonly mtimeMs: number; |
| 94 | } |
| 95 | |
| 96 | /** |
| 97 | * Clones a private, Git-written checkout with APFS copy-on-write, then rewrites the index's stat |
| 98 | * data so Git trusts the clones without re-reading them. An older cached checkout is moved to the |
| 99 | * requested tree with `read-tree -m -u`. Undefined where the platform cannot clone a folder. |
| 100 | */ |
| 101 | export const makeWorktreeCheckoutCache = Effect.fn("WorktreeCheckoutCache.make")(function* (input: { |
| 102 | readonly cacheDir: string; |
| 103 | readonly execute: GitVcsDriver.GitVcsDriver["Service"]["execute"]; |
| 104 | }) { |
| 105 | const clone = yield* CheckoutCloner.CheckoutCloner; |
| 106 | if (clone === undefined) return undefined; |
| 107 | const fileSystem = yield* FileSystem.FileSystem; |
| 108 | const path = yield* Path.Path; |
| 109 | const scope = yield* Effect.scope; |
| 110 | const homeDirectory = yield* HostProcess.HomeDirectory; |
| 111 | const environment = yield* HostProcess.Environment; |
| 112 | const builds = yield* Semaphore.make(1); |
| 113 | const pendingBuilds = new Set<string>(); |
| 114 | // Bumped by `clear`, so a build that started before it does not publish its result. |
| 115 | let generation = 0; |
| 116 | const { cacheDir, execute } = input; |
| 117 | |
| 118 | const git = ( |
| 119 | operation: string, |
| 120 | cwd: string, |
| 121 | args: ReadonlyArray<string>, |
| 122 | env?: NodeJS.ProcessEnv, |
| 123 | ) => |
| 124 | execute({ |
| 125 | operation: `WorktreeCheckoutCache.git.${operation}`, |
| 126 | cwd, |
| 127 | args, |
| 128 | allowNonZeroExit: true, |
| 129 | timeoutMs: null, |
| 130 | ...(env ? { env } : {}), |
| 131 | }).pipe( |
| 132 | Effect.map((result) => (result.exitCode === 0 ? result.stdout : null)), |
| 133 | Effect.orElseSucceed(() => null), |
| 134 | ); |
| 135 | |
| 136 | const gitSupported = yield* Effect.cached( |
| 137 | git("version", homeDirectory, ["version"]).pipe( |
| 138 | Effect.map((stdout) => { |
| 139 | const match = stdout?.match(/git version (\d+)\.(\d+)/); |
| 140 | if (!match) return false; |
| 141 | const [major, minor] = [Number(match[1]), Number(match[2])]; |
| 142 | return major > MIN_GIT[0] || (major === MIN_GIT[0] && minor >= MIN_GIT[1]); |
| 143 | }), |
| 144 | ), |
| 145 | ); |
| 146 | |
| 147 | const describe = Effect.fn("WorktreeCheckoutCache.describe")(function* ( |
| 148 | cwd: string, |
| 149 | rev: string, |
| 150 | ) { |
| 151 | const stdout = yield* git("describe", cwd, [ |
| 152 | "rev-parse", |
| 153 | "--path-format=absolute", |
| 154 | "--git-common-dir", |
| 155 | "--show-object-format", |
| 156 | "--git-path", |
| 157 | "hooks/post-index-change", |
| 158 | `${rev}^{commit}`, |
| 159 | `${rev}^{tree}`, |
| 160 | ]); |
| 161 | const [commonDir, objectFormat, indexHook, commit, tree] = stdout?.trim().split("\n") ?? []; |
| 162 | if ( |
| 163 | !commonDir || |
| 164 | objectFormat !== "sha1" || |
| 165 | !indexHook || |
| 166 | !commit || |
| 167 | !tree || |
| 168 | !TREE_ID.test(tree) |
| 169 | ) { |
| 170 | return null; |
| 171 | } |
| 172 | const realCommonDir = yield* fileSystem |
| 173 | .realPath(commonDir) |
| 174 | .pipe(Effect.orElseSucceed(() => commonDir)); |
| 175 | const repositoryKey = NodeCrypto.createHash("sha256") |
| 176 | .update(realCommonDir) |
| 177 | .digest("hex") |
| 178 | .slice(0, 16); |
| 179 | return { |
| 180 | commonDir, |
| 181 | commit, |
| 182 | tree, |
| 183 | indexHook: path.resolve(cwd, indexHook), |
| 184 | bucket: path.join(cacheDir, repositoryKey), |
| 185 | }; |
| 186 | }); |
| 187 | |
| 188 | // Empty when `GIT_ATTR_NOSYSTEM` turns the system attributes file off. |
| 189 | const systemAttributePaths = yield* Effect.cached( |
| 190 | git("systemAttributes", homeDirectory, ["var", "GIT_ATTR_SYSTEM"]).pipe( |
| 191 | Effect.map((stdout) => (stdout?.trim() ? [stdout.trim()] : [])), |
| 192 | ), |
| 193 | ); |
| 194 | |
| 195 | // Null for setups the cache cannot reproduce, such as sparse checkouts. |
| 196 | const fingerprint = Effect.fn("WorktreeCheckoutCache.fingerprint")(function* ( |
| 197 | cwd: string, |
| 198 | commonDir: string, |
| 199 | ) { |
| 200 | const config = |
| 201 | (yield* git("fingerprint", cwd, ["config", "--null", "--get-regexp", CHECKOUT_CONFIG])) ?? ""; |
| 202 | const pairs = config |
| 203 | .split("\0") |
| 204 | .filter(Boolean) |
| 205 | .map((pair) => { |
| 206 | const newline = pair.indexOf("\n"); |
| 207 | return newline < 0 |
| 208 | ? ([pair, ""] as const) |
| 209 | : ([pair.slice(0, newline), pair.slice(newline + 1)] as const); |
| 210 | }); |
| 211 | const sparse = pairs.findLast(([key]) => key === "core.sparsecheckout")?.[1]; |
| 212 | if (sparse !== undefined && !GIT_FALSE.has(sparse.toLowerCase())) return null; |
| 213 | const attributesFile = pairs |
| 214 | .findLast(([key]) => key === "core.attributesfile")?.[1] |
| 215 | ?.replace(/^~(?=$|\/)/, homeDirectory); |
| 216 | // Git reads a relative path from each checkout, which differs between the cache and the worktree. |
| 217 | if (attributesFile !== undefined && !path.isAbsolute(attributesFile)) return null; |
| 218 | const attributePaths = [ |
| 219 | ...(yield* systemAttributePaths), |
| 220 | path.join(commonDir, "info", "attributes"), |
| 221 | attributesFile ?? |
| 222 | path.join( |
| 223 | environment.XDG_CONFIG_HOME || path.join(homeDirectory, ".config"), |
| 224 | "git", |
| 225 | "attributes", |
| 226 | ), |
| 227 | ]; |
| 228 | const attributes = yield* Effect.forEach(attributePaths, (file) => |
| 229 | fileSystem.readFileString(file).pipe(Effect.orElseSucceed(() => "")), |
| 230 | ); |
| 231 | const hash = NodeCrypto.createHash("sha256").update(config); |
| 232 | for (const contents of attributes) hash.update("\0").update(contents); |
| 233 | return hash.digest("hex").slice(0, 16); |
| 234 | }); |
| 235 | |
| 236 | const listEntries = Effect.fn("WorktreeCheckoutCache.listEntries")(function* (bucket: string) { |
| 237 | const names = yield* fileSystem.readDirectory(bucket).pipe(Effect.orElseSucceed(() => [])); |
| 238 | const entries = yield* Effect.forEach( |
| 239 | names.flatMap((name) => { |
| 240 | const match = ENTRY_NAME.exec(name); |
| 241 | return match ? [{ name, tree: match[1]!, fingerprint: match[2]! }] : []; |
| 242 | }), |
| 243 | ({ name, tree, fingerprint: entryFingerprint }) => { |
| 244 | const directory = path.join(bucket, name); |
| 245 | return fileSystem.stat(directory).pipe( |
| 246 | Effect.map((info): CacheEntry => ({ |
| 247 | directory, |
| 248 | tree, |
| 249 | fingerprint: entryFingerprint, |
| 250 | mtimeMs: Option.match(info.mtime, { |
| 251 | onNone: () => 0, |
| 252 | onSome: (date) => date.getTime(), |
| 253 | }), |
| 254 | })), |
| 255 | Effect.option, |
| 256 | ); |
| 257 | }, |
| 258 | { concurrency: 8 }, |
| 259 | ); |
| 260 | return entries |
| 261 | .flatMap((entry) => (entry._tag === "Some" ? [entry.value] : [])) |
| 262 | .toSorted((left, right) => right.mtimeMs - left.mtimeMs); |
| 263 | }); |
| 264 | |
| 265 | const touch = (target: string) => |
| 266 | Clock.currentTimeMillis.pipe( |
| 267 | Effect.flatMap((now) => |
| 268 | Effect.tryPromise({ |
| 269 | try: () => NodeFSP.utimes(target, now / 1000, now / 1000), |
| 270 | catch: (cause) => |
| 271 | new WorktreeCheckoutCacheError({ detail: `could not touch ${target}: ${cause}` }), |
| 272 | }), |
| 273 | ), |
| 274 | Effect.ignore, |
| 275 | ); |
| 276 | |
| 277 | // Untraced: one stat per tracked file, already covered by the caller's span. |
| 278 | const statEntries = Effect.fnUntraced(function* (root: string, paths: ReadonlyArray<string>) { |
| 279 | return yield* Effect.tryPromise({ |
| 280 | try: async () => { |
| 281 | const stats: Array<IndexEntryStat | undefined> = []; |
| 282 | for (let start = 0; start < paths.length; start += STAT_BATCH) { |
| 283 | const batch = await Promise.all( |
| 284 | paths.slice(start, start + STAT_BATCH).map((entryPath) => |
| 285 | NodeFSP.lstat(`${root}/${entryPath}`, { bigint: true }).then( |
| 286 | indexEntryStat, |
| 287 | (error) => { |
| 288 | if (error?.code === "ENOENT") return undefined; |
| 289 | throw error; |
| 290 | }, |
| 291 | ), |
| 292 | ), |
| 293 | ); |
| 294 | stats.push(...batch); |
| 295 | } |
| 296 | return stats; |
| 297 | }, |
| 298 | catch: (cause) => |
| 299 | new WorktreeCheckoutCacheError({ detail: `could not stat ${root}: ${cause}` }), |
| 300 | }); |
| 301 | }); |
| 302 | |
| 303 | /** |
| 304 | * Writes `target` from the index at `source` with stat data from the files under `root`, and |
| 305 | * returns what it read. Null when the index or the files cannot be trusted. |
| 306 | */ |
| 307 | const restate = Effect.fn("WorktreeCheckoutCache.restate")(function* (input: { |
| 308 | readonly source: string; |
| 309 | readonly target: string; |
| 310 | readonly root: string; |
| 311 | readonly verify: boolean; |
| 312 | }) { |
| 313 | const parsed = parseIndex(yield* fileSystem.readFile(input.source)); |
| 314 | if (parsed === null) return null; |
| 315 | const paths = parsed.entries.map((entry) => entry.path); |
| 316 | const stats = yield* statEntries(input.root, paths); |
| 317 | const restated = restatIndex(parsed, stats, { verify: input.verify }); |
| 318 | if (restated === null) return null; |
| 319 | const temporary = `${input.target}.t3-${NodeCrypto.randomUUID()}`; |
| 320 | yield* Effect.uninterruptible( |
| 321 | fileSystem |
| 322 | .writeFile(temporary, restated) |
| 323 | .pipe(Effect.andThen(fileSystem.rename(temporary, input.target))), |
| 324 | ); |
| 325 | return { paths, stats }; |
| 326 | }); |
| 327 | |
| 328 | // Untraced: one stat per folder, already covered by the caller's span. |
| 329 | const directoryTimes = Effect.fnUntraced(function* ( |
| 330 | root: string, |
| 331 | directories: ReadonlyArray<string>, |
| 332 | ) { |
| 333 | return yield* Effect.tryPromise({ |
| 334 | try: () => |
| 335 | Promise.all( |
| 336 | directories.map((directory) => |
| 337 | NodeFSP.lstat(directory === "" ? root : `${root}/${directory}`, { bigint: true }).then( |
| 338 | (stat) => String(stat.mtimeNs), |
| 339 | () => "missing", |
| 340 | ), |
| 341 | ), |
| 342 | ), |
| 343 | catch: (cause) => |
| 344 | new WorktreeCheckoutCacheError({ detail: `could not stat folders in ${root}: ${cause}` }), |
| 345 | }); |
| 346 | }); |
| 347 | |
| 348 | // Adding or removing a file changes its folder's mtime, so this catches files the index lacks. |
| 349 | const recordDirectories = Effect.fn("WorktreeCheckoutCache.recordDirectories")(function* ( |
| 350 | entryDirectory: string, |
| 351 | recorded: { |
| 352 | readonly paths: ReadonlyArray<string>; |
| 353 | readonly stats: ReadonlyArray<IndexEntryStat | undefined>; |
| 354 | }, |
| 355 | ) { |
| 356 | const directories = parentDirectories(recorded.paths); |
| 357 | for (const [index, entryPath] of recorded.paths.entries()) { |
| 358 | if (recorded.stats[index]?.kind === "directory") directories.add(entryPath); |
| 359 | } |
| 360 | // Folders whose names differ only in case share one spelling on disk, which depends on history. |
| 361 | const folded = new Set([...directories].map((directory) => foldCase(directory))); |
| 362 | if (folded.size !== directories.size) { |
| 363 | return yield* new WorktreeCheckoutCacheError({ detail: "folder names differ only in case" }); |
| 364 | } |
| 365 | const list = [...directories]; |
| 366 | // Backdated, so any later change moves the time even on a filesystem with a coarse clock. |
| 367 | const root = path.join(entryDirectory, "files"); |
| 368 | yield* Effect.tryPromise({ |
| 369 | try: () => |
| 370 | Promise.all( |
| 371 | list.map((directory) => |
| 372 | NodeFSP.utimes(directory === "" ? root : `${root}/${directory}`, 1_000_000, 1_000_000), |
| 373 | ), |
| 374 | ), |
| 375 | catch: (cause) => |
| 376 | new WorktreeCheckoutCacheError({ detail: `could not backdate cached folders: ${cause}` }), |
| 377 | }); |
| 378 | const times = yield* directoryTimes(root, list); |
| 379 | yield* fileSystem.writeFileString( |
| 380 | path.join(entryDirectory, "directories"), |
| 381 | list.map((directory, index) => `${times[index]}\0${directory}\0`).join(""), |
| 382 | ); |
| 383 | }); |
| 384 | |
| 385 | const directoriesUnchanged = Effect.fn("WorktreeCheckoutCache.directoriesUnchanged")( |
| 386 | function* (entry: CacheEntry) { |
| 387 | const recorded = (yield* fileSystem.readFileString(path.join(entry.directory, "directories"))) |
| 388 | .split("\0") |
| 389 | .slice(0, -1); |
| 390 | const times = recorded.filter((_, index) => index % 2 === 0); |
| 391 | const directories = recorded.filter((_, index) => index % 2 === 1); |
| 392 | const current = yield* directoryTimes(path.join(entry.directory, "files"), directories); |
| 393 | return directories.length > 0 && current.every((time, index) => time === times[index]); |
| 394 | }, |
| 395 | (effect) => Effect.orElseSucceed(effect, () => false), |
| 396 | ); |
| 397 | |
| 398 | const changedPaths = Effect.fn("WorktreeCheckoutCache.changedPaths")(function* ( |
| 399 | cwd: string, |
| 400 | from: string, |
| 401 | to: string, |
| 402 | ) { |
| 403 | const stdout = yield* git("changes", cwd, [ |
| 404 | "diff-tree", |
| 405 | "-r", |
| 406 | "-z", |
| 407 | "--name-status", |
| 408 | "--no-renames", |
| 409 | from, |
| 410 | to, |
| 411 | ]); |
| 412 | if (stdout === null) return null; |
| 413 | const fields = stdout.split("\0"); |
| 414 | const added: Array<string> = []; |
| 415 | const deleted: Array<string> = []; |
| 416 | // Added or changed: what `read-tree -m -u` writes itself. |
| 417 | const written: Array<string> = []; |
| 418 | let count = 0; |
| 419 | let attributes = false; |
| 420 | for (let index = 0; index + 1 < fields.length; index += 2) { |
| 421 | const [status, file] = [fields[index]!, fields[index + 1]!]; |
| 422 | count++; |
| 423 | // `read-tree -m -u` leaves unchanged files alone, even when their attributes changed. |
| 424 | if (file === ".gitattributes" || file.endsWith("/.gitattributes")) attributes = true; |
| 425 | if (status === "A") added.push(file); |
| 426 | if (status === "D") deleted.push(file); |
| 427 | else written.push(file); |
| 428 | } |
| 429 | return { count, attributes, added, deleted, written }; |
| 430 | }); |
| 431 | |
| 432 | // A smudge filter's output can depend on the environment and the network, as with Git LFS. |
| 433 | const usesFilter = Effect.fn("WorktreeCheckoutCache.usesFilter")(function* ( |
| 434 | cwd: string, |
| 435 | bucket: string, |
| 436 | tree: string, |
| 437 | ) { |
| 438 | // An index of the tree lets `check-attr` read its `.gitattributes` on any Git version. |
| 439 | const index = path.join(bucket, `.attributes-${NodeCrypto.randomUUID()}`); |
| 440 | const env = { GIT_INDEX_FILE: index }; |
| 441 | const run = (args: ReadonlyArray<string>, stdin?: string) => |
| 442 | execute({ |
| 443 | operation: "WorktreeCheckoutCache.git.filters", |
| 444 | cwd, |
| 445 | args, |
| 446 | env, |
| 447 | allowNonZeroExit: true, |
| 448 | maxOutputBytes: MAX_LIST_BYTES, |
| 449 | ...(stdin === undefined ? {} : { stdin }), |
| 450 | }).pipe(Effect.map((result) => (result.exitCode === 0 ? result.stdout : null))); |
| 451 | return yield* Effect.gen(function* () { |
| 452 | if ((yield* run([...CACHE_INDEX_CONFIG, "read-tree", tree])) === null) return true; |
| 453 | const files = yield* run(["ls-files", "-z"]); |
| 454 | if (files === null) return true; |
| 455 | const output = yield* run(["check-attr", "--cached", "--all", "-z", "--stdin"], files); |
| 456 | return output === null || namesFilter(output); |
| 457 | }).pipe( |
| 458 | Effect.orElseSucceed(() => true), |
| 459 | Effect.ensuring(fileSystem.remove(index).pipe(Effect.ignore)), |
| 460 | ); |
| 461 | }); |
| 462 | |
| 463 | // Attributes come from the cloned `.gitattributes` files, which the move leaves unchanged. |
| 464 | const writesWithFilter = Effect.fn("WorktreeCheckoutCache.writesWithFilter")(function* ( |
| 465 | cwd: string, |
| 466 | files: ReadonlyArray<string>, |
| 467 | ) { |
| 468 | if (files.length === 0) return false; |
| 469 | const output = yield* execute({ |
| 470 | operation: "WorktreeCheckoutCache.git.filters", |
| 471 | cwd, |
| 472 | args: ["check-attr", "--all", "-z", "--stdin"], |
| 473 | stdin: files.map((file) => `${file}\0`).join(""), |
| 474 | allowNonZeroExit: true, |
| 475 | maxOutputBytes: MAX_LIST_BYTES, |
| 476 | }).pipe( |
| 477 | Effect.map((result) => (result.exitCode === 0 ? result.stdout : null)), |
| 478 | Effect.orElseSucceed(() => null), |
| 479 | ); |
| 480 | return output === null || namesFilter(output); |
| 481 | }); |
| 482 | |
| 483 | // Renamed first so nothing clones a half-deleted checkout; deleting a large one takes seconds. |
| 484 | const discard = Effect.fn("WorktreeCheckoutCache.discard")( |
| 485 | function* (target: string) { |
| 486 | const trashed = `${target}.trash-${NodeCrypto.randomUUID()}`; |
| 487 | yield* fileSystem.rename(target, trashed); |
| 488 | yield* fileSystem.remove(trashed, { recursive: true }); |
| 489 | }, |
| 490 | // Uninterruptible: deleting is not resumable, and a half-deleted folder must not be left named. |
| 491 | (effect) => Effect.ignore(Effect.uninterruptible(effect)), |
| 492 | ); |
| 493 | |
| 494 | const gitDirOf = Effect.fn("WorktreeCheckoutCache.gitDirOf")(function* (worktreePath: string) { |
| 495 | const contents = yield* fileSystem.readFileString(path.join(worktreePath, ".git")); |
| 496 | const gitDir = contents.match(/^gitdir: (.+)$/m)?.[1]; |
| 497 | if (gitDir === undefined) { |
| 498 | return yield* new WorktreeCheckoutCacheError({ |
| 499 | detail: `${worktreePath} is not a linked worktree`, |
| 500 | }); |
| 501 | } |
| 502 | return path.resolve(worktreePath, gitDir.trim()); |
| 503 | }); |
| 504 | |
| 505 | /** |
| 506 | * Creates the worktree from the cache. False means nothing was created and the caller should |
| 507 | * run `git worktree add`; any partial worktree has been removed again. |
| 508 | */ |
| 509 | const create = Effect.fn("WorktreeCheckoutCache.create")(function* (input: { |
| 510 | readonly cwd: string; |
| 511 | readonly worktreePath: string; |
| 512 | readonly refName: string; |
| 513 | readonly checkoutWorkers: string; |
| 514 | readonly lockReason?: string | undefined; |
| 515 | }) { |
| 516 | if (!(yield* gitSupported)) return false; |
| 517 | const worktreePath = path.resolve(input.cwd, input.worktreePath); |
| 518 | const requested = yield* describe(input.cwd, input.refName); |
| 519 | if (requested === null) return false; |
| 520 | const entries = yield* listEntries(requested.bucket); |
| 521 | if (entries.length === 0) return false; |
| 522 | // A clone only works within one volume. |
| 523 | const parent = path.dirname(worktreePath); |
| 524 | yield* fileSystem.makeDirectory(parent, { recursive: true }).pipe(Effect.ignore); |
| 525 | const [bucketInfo, parentInfo] = yield* Effect.all([ |
| 526 | fileSystem.stat(requested.bucket), |
| 527 | fileSystem.stat(parent), |
| 528 | ]).pipe(Effect.orElseSucceed(() => [null, null] as const)); |
| 529 | if (bucketInfo === null || parentInfo === null || bucketInfo.dev !== parentInfo.dev) { |
| 530 | return false; |
| 531 | } |
| 532 | |
| 533 | const populate = Effect.gen(function* () { |
| 534 | // What Git checked out, which can differ from `requested` when a tag shares the branch name. |
| 535 | const target = yield* describe(worktreePath, "HEAD"); |
| 536 | if (target === null || (yield* fileSystem.exists(target.indexHook))) return null; |
| 537 | const worktreeFingerprint = yield* fingerprint(worktreePath, target.commonDir); |
| 538 | if (worktreeFingerprint === null) return null; |
| 539 | const compatible = entries.filter((entry) => entry.fingerprint === worktreeFingerprint); |
| 540 | let base = compatible.find((entry) => entry.tree === target.tree); |
| 541 | let changes: Effect.Success<ReturnType<typeof changedPaths>> = null; |
| 542 | for (const entry of base === undefined ? compatible : []) { |
| 543 | changes = yield* changedPaths(worktreePath, entry.tree, target.tree); |
| 544 | if (changes !== null && !changes.attributes) { |
| 545 | base = entry; |
| 546 | break; |
| 547 | } |
| 548 | } |
| 549 | if (base === undefined) return null; |
| 550 | // A cached file changed after Git wrote it; never hand it out again. |
| 551 | const reject = (entry: CacheEntry) => |
| 552 | Effect.logWarning("worktree checkout cache entry changed; discarding it", { |
| 553 | directory: entry.directory, |
| 554 | }).pipe(Effect.andThen(discard(entry.directory)), Effect.as(null)); |
| 555 | if (!(yield* directoriesUnchanged(base))) return yield* reject(base); |
| 556 | yield* Effect.all([touch(base.directory), touch(requested.bucket)], { discard: true }); |
| 557 | |
| 558 | const files = path.join(base.directory, "files"); |
| 559 | // The worktree folder already holds Git's `.git` file, so its entries are cloned one by one. |
| 560 | yield* Effect.forEach( |
| 561 | yield* fileSystem.readDirectory(files), |
| 562 | (name) => clone(path.join(files, name), path.join(worktreePath, name)), |
| 563 | { discard: true }, |
| 564 | ); |
| 565 | // Checked again: a file added during the clone would otherwise come along unnoticed. |
| 566 | if (!(yield* directoriesUnchanged(base))) return yield* reject(base); |
| 567 | // A tracked hooks folder only exists once the files do; Git would have run this hook. |
| 568 | if (yield* fileSystem.exists(target.indexHook)) return null; |
| 569 | const restated = yield* restate({ |
| 570 | source: path.join(base.directory, "index"), |
| 571 | target: path.join(yield* gitDirOf(worktreePath), "index"), |
| 572 | root: worktreePath, |
| 573 | verify: true, |
| 574 | }); |
| 575 | if (restated === null) return yield* reject(base); |
| 576 | if (base.tree === target.tree || changes === null) return target; |
| 577 | // On a case-insensitive volume an added path can land on a file the cached tree kept. |
| 578 | const deleted = new Set(changes.deleted); |
| 579 | const kept = restated.paths.filter((file) => !deleted.has(file)); |
| 580 | const names = new Set(kept.map((file) => foldCase(file))); |
| 581 | // A new folder whose name differs only in case would keep the cached spelling. |
| 582 | const keptFolders = parentDirectories(kept); |
| 583 | for (const [index, file] of restated.paths.entries()) { |
| 584 | if (restated.stats[index]?.kind === "directory" && !deleted.has(file)) |
| 585 | keptFolders.add(file); |
| 586 | } |
| 587 | const spellings = new Map( |
| 588 | [...keptFolders].map((directory) => [foldCase(directory), directory]), |
| 589 | ); |
| 590 | for (const file of changes.added) { |
| 591 | if (names.has(foldCase(file))) return null; |
| 592 | names.add(foldCase(file)); |
| 593 | for (const directory of parentDirectories([file])) { |
| 594 | const existing = spellings.get(foldCase(directory)); |
| 595 | if (existing !== undefined && existing !== directory) return null; |
| 596 | spellings.set(foldCase(directory), directory); |
| 597 | } |
| 598 | } |
| 599 | // A smudge filter run here would see the cached index, not the one Git builds from scratch. |
| 600 | if (yield* writesWithFilter(worktreePath, changes.written)) return null; |
| 601 | // Hooks off: a hook the move adds runs with Git's checkout environment there, so its |
| 602 | // presence afterwards means the caller falls back to that checkout. |
| 603 | const moved = yield* git("update", worktreePath, [ |
| 604 | "-c", |
| 605 | `checkout.workers=${input.checkoutWorkers}`, |
| 606 | "-c", |
| 607 | "core.hooksPath=/dev/null", |
| 608 | "read-tree", |
| 609 | "-m", |
| 610 | "-u", |
| 611 | base.tree, |
| 612 | target.tree, |
| 613 | ]); |
| 614 | if (moved === null || (yield* fileSystem.exists(target.indexHook))) return null; |
| 615 | return target; |
| 616 | }); |
| 617 | const rollback = ( |
| 618 | input.lockReason |
| 619 | ? git("unlockRollback", input.cwd, ["worktree", "unlock", "--", worktreePath]) |
| 620 | : Effect.void |
| 621 | ).pipe( |
| 622 | Effect.andThen( |
| 623 | git("rollback", input.cwd, ["worktree", "remove", "--force", "--", worktreePath]), |
| 624 | ), |
| 625 | Effect.flatMap((removed) => |
| 626 | removed === null |
| 627 | ? Effect.logWarning("could not remove a partially created worktree", { worktreePath }) |
| 628 | : Effect.void, |
| 629 | ), |
| 630 | ); |
| 631 | // Registration and the rollback that undoes it can't be separated by an interruption. |
| 632 | const target = yield* Effect.uninterruptibleMask((restore) => |
| 633 | Effect.gen(function* () { |
| 634 | const registered = yield* git("register", input.cwd, [ |
| 635 | "worktree", |
| 636 | "add", |
| 637 | "--no-checkout", |
| 638 | ...(input.lockReason ? ["--lock", "--reason", input.lockReason] : []), |
| 639 | "--", |
| 640 | worktreePath, |
| 641 | input.refName, |
| 642 | ]); |
| 643 | if (registered === null) return null; |
| 644 | return yield* restore(populate).pipe( |
| 645 | Effect.catch((cause) => |
| 646 | Effect.logWarning("worktree checkout cache failed; using git worktree add", { |
| 647 | cause, |
| 648 | }).pipe(Effect.as(null)), |
| 649 | ), |
| 650 | Effect.onExit((exit) => |
| 651 | Exit.isSuccess(exit) && exit.value !== null ? Effect.void : rollback, |
| 652 | ), |
| 653 | ); |
| 654 | }), |
| 655 | ); |
| 656 | if (target === null) return false; |
| 657 | |
| 658 | // `git worktree add --no-checkout` skips this hook; a normal add runs it. |
| 659 | const hook = yield* execute({ |
| 660 | operation: "GitVcsDriver.createWorktree.postCheckout", |
| 661 | cwd: worktreePath, |
| 662 | args: [ |
| 663 | "hook", |
| 664 | "run", |
| 665 | "--ignore-missing", |
| 666 | "post-checkout", |
| 667 | "--", |
| 668 | NULL_OID, |
| 669 | target.commit, |
| 670 | "1", |
| 671 | ], |
| 672 | env: { LANGUAGE: "C" }, |
| 673 | allowNonZeroExit: true, |
| 674 | timeoutMs: null, |
| 675 | }); |
| 676 | if (hook.exitCode !== 0) { |
| 677 | return yield* new GitCommandError({ |
| 678 | operation: "GitVcsDriver.createWorktree", |
| 679 | command: "git worktree add", |
| 680 | cwd: input.cwd, |
| 681 | detail: "git worktree add failed: the post-checkout hook failed", |
| 682 | exitCode: hook.exitCode, |
| 683 | stderrLength: hook.stderr.length, |
| 684 | }); |
| 685 | } |
| 686 | return true; |
| 687 | }); |
| 688 | |
| 689 | const removeIfAbandoned = Effect.fn("WorktreeCheckoutCache.removeIfAbandoned")( |
| 690 | function* (target: string, maxAgeMs: number) { |
| 691 | const info = yield* fileSystem.stat(target); |
| 692 | const modified = Option.getOrUndefined(info.mtime)?.getTime() ?? 0; |
| 693 | if ((yield* Clock.currentTimeMillis) - modified > maxAgeMs) { |
| 694 | yield* fileSystem.remove(target, { recursive: true }); |
| 695 | } |
| 696 | }, |
| 697 | (effect) => Effect.ignore(effect), |
| 698 | ); |
| 699 | |
| 700 | /** Caches the tree checked out in `worktreePath`, unless a cached checkout is already close to it. */ |
| 701 | const build = Effect.fn("WorktreeCheckoutCache.build")(function* ( |
| 702 | worktreePath: string, |
| 703 | // From before the worktree was checked out, so a `clear` during the checkout also counts. |
| 704 | generationAtStart: number, |
| 705 | ) { |
| 706 | if (!(yield* gitSupported)) return; |
| 707 | const target = yield* describe(worktreePath, "HEAD"); |
| 708 | if (target === null) return; |
| 709 | const entryFingerprint = yield* fingerprint(worktreePath, target.commonDir); |
| 710 | if (entryFingerprint === null) return; |
| 711 | const entryName = `${target.tree}-${entryFingerprint}`; |
| 712 | yield* fileSystem.makeDirectory(target.bucket, { recursive: true }); |
| 713 | const entries = yield* listEntries(target.bucket); |
| 714 | const compatible = entries.filter((entry) => entry.fingerprint === entryFingerprint); |
| 715 | if (compatible.some((entry) => entry.tree === target.tree)) return; |
| 716 | // Skip when a cached checkout is close enough to move forward cheaply. |
| 717 | let base: CacheEntry | undefined; |
| 718 | for (const entry of compatible) { |
| 719 | const changes = yield* changedPaths(worktreePath, entry.tree, target.tree); |
| 720 | if (changes === null || changes.attributes) continue; |
| 721 | if (changes.count < REBUILD_AFTER_CHANGED_FILES) return; |
| 722 | base ??= entry; |
| 723 | } |
| 724 | if (yield* usesFilter(worktreePath, target.bucket, target.tree)) return; |
| 725 | |
| 726 | for (const name of yield* fileSystem.readDirectory(target.bucket)) { |
| 727 | if (!ENTRY_NAME.test(name)) { |
| 728 | const leftover = path.join(target.bucket, name); |
| 729 | yield* removeIfAbandoned( |
| 730 | leftover, |
| 731 | name.includes(".trash-") ? ABANDONED_TRASH_MS : ABANDONED_STAGING_MS, |
| 732 | ); |
| 733 | } |
| 734 | } |
| 735 | const staging = path.join(target.bucket, `.staging-${NodeCrypto.randomUUID()}`); |
| 736 | const stagingFiles = path.join(staging, "files"); |
| 737 | const stagingIndex = path.join(staging, "index"); |
| 738 | yield* fileSystem.makeDirectory(staging); |
| 739 | const withIndex = (args: ReadonlyArray<string>) => |
| 740 | git("build", worktreePath, [...CACHE_INDEX_CONFIG, `--work-tree=${stagingFiles}`, ...args], { |
| 741 | GIT_INDEX_FILE: stagingIndex, |
| 742 | }); |
| 743 | |
| 744 | yield* Effect.gen(function* () { |
| 745 | let built = false; |
| 746 | if (base !== undefined && (yield* directoriesUnchanged(base))) { |
| 747 | const from = base; |
| 748 | built = yield* Effect.gen(function* () { |
| 749 | yield* Effect.uninterruptible(clone(path.join(from.directory, "files"), stagingFiles)); |
| 750 | if (!(yield* directoriesUnchanged(from))) return false; |
| 751 | const restated = yield* restate({ |
| 752 | source: path.join(from.directory, "index"), |
| 753 | target: stagingIndex, |
| 754 | root: stagingFiles, |
| 755 | verify: true, |
| 756 | }); |
| 757 | if (restated === null) return false; |
| 758 | return (yield* withIndex(["read-tree", "-m", "-u", from.tree, target.tree])) !== null; |
| 759 | }).pipe(Effect.orElseSucceed(() => false)); |
| 760 | if (!built) { |
| 761 | yield* fileSystem.remove(stagingFiles, { recursive: true }).pipe(Effect.ignore); |
| 762 | yield* fileSystem.remove(stagingIndex).pipe(Effect.ignore); |
| 763 | } |
| 764 | } |
| 765 | if (!built) { |
| 766 | yield* fileSystem.makeDirectory(stagingFiles); |
| 767 | if ((yield* withIndex(["read-tree", "--reset", "-u", target.tree])) === null) { |
| 768 | return yield* new WorktreeCheckoutCacheError({ detail: "git read-tree failed" }); |
| 769 | } |
| 770 | } |
| 771 | // Record the files' own stat data; Git leaves racily written entries for a later re-read. |
| 772 | const recorded = yield* restate({ |
| 773 | source: stagingIndex, |
| 774 | target: stagingIndex, |
| 775 | root: stagingFiles, |
| 776 | verify: false, |
| 777 | }); |
| 778 | if (recorded === null) { |
| 779 | return yield* new WorktreeCheckoutCacheError({ |
| 780 | detail: "the cache index has an unsupported format or colliding paths", |
| 781 | }); |
| 782 | } |
| 783 | yield* recordDirectories(staging, recorded); |
| 784 | if (generation !== generationAtStart) { |
| 785 | return yield* new WorktreeCheckoutCacheError({ detail: "the cache was cleared" }); |
| 786 | } |
| 787 | yield* Effect.uninterruptible( |
| 788 | fileSystem.rename(staging, path.join(target.bucket, entryName)), |
| 789 | ); |
| 790 | }).pipe(Effect.onError(() => discard(staging))); |
| 791 | |
| 792 | yield* touch(target.bucket); |
| 793 | yield* Effect.forEach( |
| 794 | (yield* listEntries(target.bucket)).slice(KEEP_PER_REPOSITORY), |
| 795 | (entry) => discard(entry.directory), |
| 796 | { discard: true }, |
| 797 | ); |
| 798 | const now = yield* Clock.currentTimeMillis; |
| 799 | for (const name of yield* fileSystem.readDirectory(cacheDir)) { |
| 800 | const bucket = path.join(cacheDir, name); |
| 801 | if (bucket === target.bucket) continue; |
| 802 | if (name.includes(".trash-")) { |
| 803 | yield* removeIfAbandoned(bucket, ABANDONED_TRASH_MS); |
| 804 | continue; |
| 805 | } |
| 806 | const info = yield* fileSystem.stat(bucket).pipe(Effect.option); |
| 807 | const used = info.pipe( |
| 808 | Option.flatMap((value) => value.mtime), |
| 809 | Option.map((date) => date.getTime()), |
| 810 | ); |
| 811 | if (Option.isSome(used) && now - used.value > UNUSED_REPOSITORY_MS) { |
| 812 | yield* discard(bucket); |
| 813 | } |
| 814 | } |
| 815 | }); |
| 816 | |
| 817 | /** Caches the worktree's checkout in the background, so the next worktree can clone it. */ |
| 818 | const schedule = (worktreePath: string, generationAtStart: number) => |
| 819 | Effect.suspend(() => { |
| 820 | if (pendingBuilds.has(worktreePath)) return Effect.void; |
| 821 | pendingBuilds.add(worktreePath); |
| 822 | return build(worktreePath, generationAtStart).pipe( |
| 823 | builds.withPermits(1), |
| 824 | Effect.ignoreCause({ log: true }), |
| 825 | Effect.ensuring(Effect.sync(() => pendingBuilds.delete(worktreePath))), |
| 826 | Effect.forkIn(scope), |
| 827 | Effect.asVoid, |
| 828 | ); |
| 829 | }); |
| 830 | |
| 831 | /** Deletes every cached checkout in the background; builds already running are dropped. */ |
| 832 | const clear = Effect.suspend(() => { |
| 833 | generation++; |
| 834 | return fileSystem.exists(cacheDir); |
| 835 | }).pipe( |
| 836 | Effect.flatMap((exists) => |
| 837 | exists |
| 838 | ? discard(cacheDir).pipe(builds.withPermits(1), Effect.forkIn(scope), Effect.asVoid) |
| 839 | : Effect.void, |
| 840 | ), |
| 841 | Effect.ignore, |
| 842 | ); |
| 843 | |
| 844 | /** Pass to `schedule` from before a checkout starts. */ |
| 845 | const currentGeneration = Effect.sync(() => generation); |
| 846 | |
| 847 | return { create, build, schedule, clear, currentGeneration }; |
| 848 | }); |