The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off - Comparing a file with its last capture needs bigint lstat, which FileSystem lacks. |
| 2 | import * as NodeFSP from "node:fs/promises"; |
| 3 | |
| 4 | import * as Effect from "effect/Effect"; |
| 5 | import type * as FileSystem from "effect/FileSystem"; |
| 6 | import * as Option from "effect/Option"; |
| 7 | |
| 8 | import type * as VcsDriver from "./VcsDriver.ts"; |
| 9 | import * as VcsProcess from "./VcsProcess.ts"; |
| 10 | |
| 11 | /** An untracked file's blob from the last capture, with everything that decided its content. */ |
| 12 | interface UntrackedBlob { |
| 13 | readonly path: string; |
| 14 | readonly mode: string; |
| 15 | readonly oid: string; |
| 16 | readonly size: bigint; |
| 17 | readonly stat: string; |
| 18 | readonly attributes: string; |
| 19 | } |
| 20 | |
| 21 | interface Snapshot { |
| 22 | readonly topLevel: string; |
| 23 | readonly head: string; |
| 24 | readonly config: string; |
| 25 | readonly blobs: ReadonlyArray<UntrackedBlob>; |
| 26 | /** Files hashed without a reading of their conversion state from before staging. */ |
| 27 | readonly pending: ReadonlyArray<string>; |
| 28 | } |
| 29 | |
| 30 | /** The conversion state read before staging; a blob is remembered only if it reads the same after. */ |
| 31 | interface ConversionState { |
| 32 | readonly config: string; |
| 33 | readonly attributes: ReadonlyMap<string, string | undefined>; |
| 34 | } |
| 35 | |
| 36 | export const NOTHING: { |
| 37 | readonly reused: ReadonlyArray<UntrackedBlob>; |
| 38 | readonly before: ConversionState | undefined; |
| 39 | } = { reused: [], before: undefined }; |
| 40 | |
| 41 | // Every attribute convert_to_git reads; a blob is reused only while all of them are unchanged. |
| 42 | const CONVERSION_ATTRIBUTES = [ |
| 43 | "text", |
| 44 | "eol", |
| 45 | "crlf", |
| 46 | "ident", |
| 47 | "working-tree-encoding", |
| 48 | "filter", |
| 49 | ] as const; |
| 50 | const MAX_BLOBS = 20_000; |
| 51 | // Below this, hashing the files costs less than the Git processes that check them. |
| 52 | const MIN_TOTAL_BYTES = 16n * 1024n * 1024n; |
| 53 | // A capture that staged faster than this and reused nothing has nothing worth another Git process. |
| 54 | const MIN_STAGING_MS = 100; |
| 55 | // A file must have been left alone for this long before the capture, so an edit that lands |
| 56 | // while Git hashes it cannot hide inside one timestamp tick (two seconds on FAT). |
| 57 | const SETTLED_MS = 3_000; |
| 58 | const MAX_WORKSPACES = 64; |
| 59 | const STAT_BATCH = 512; |
| 60 | const LIST_MAX_OUTPUT_BYTES = 16 * 1024 * 1024; |
| 61 | |
| 62 | const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION; |
| 63 | |
| 64 | const lstatAll = (root: string, paths: ReadonlyArray<string>) => |
| 65 | Effect.promise(async () => { |
| 66 | const stats: Array< |
| 67 | { stat: string; size: bigint; changedNs: bigint; regular: boolean } | undefined |
| 68 | > = []; |
| 69 | for (let start = 0; start < paths.length; start += STAT_BATCH) { |
| 70 | const batch = await Promise.all( |
| 71 | paths.slice(start, start + STAT_BATCH).map((entryPath) => |
| 72 | NodeFSP.lstat(`${root}/${entryPath}`, { bigint: true }).then( |
| 73 | (stat) => ({ |
| 74 | stat: `${stat.dev}:${stat.ino}:${stat.mode}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`, |
| 75 | size: stat.size, |
| 76 | changedNs: stat.mtimeNs > stat.ctimeNs ? stat.mtimeNs : stat.ctimeNs, |
| 77 | regular: stat.isFile(), |
| 78 | }), |
| 79 | () => undefined, |
| 80 | ), |
| 81 | ), |
| 82 | ); |
| 83 | stats.push(...batch); |
| 84 | } |
| 85 | return stats; |
| 86 | }); |
| 87 | |
| 88 | /** |
| 89 | * Lets a capture skip rehashing untracked files that have not changed since the last capture of |
| 90 | * the same workspace. A blob is reused only when a fresh `git add` would produce the same one: |
| 91 | * same file stat, same conversion attributes and config, still not ignored, still not tracked. |
| 92 | */ |
| 93 | export const make = ( |
| 94 | execute: VcsDriver.VcsDriver["Service"]["execute"], |
| 95 | fileSystem: FileSystem.FileSystem, |
| 96 | ) => { |
| 97 | const snapshots = new Map<string, Snapshot>(); |
| 98 | |
| 99 | // Worktree files only: an index copy of a deleted .gitattributes applies or not by staging order. |
| 100 | const readAttributes = Effect.fn("GitVcsDriver.checkpoints.readConversionAttributes")(function* ( |
| 101 | topLevel: string, |
| 102 | env: NodeJS.ProcessEnv, |
| 103 | paths: ReadonlyArray<string>, |
| 104 | ) { |
| 105 | if (paths.length === 0) return []; |
| 106 | const stdin = `${paths.join("\0")}\0`; |
| 107 | const result = yield* execute({ |
| 108 | operation, |
| 109 | cwd: topLevel, |
| 110 | args: ["check-attr", "-z", "--stdin", ...CONVERSION_ATTRIBUTES], |
| 111 | stdin, |
| 112 | env: { ...env, GIT_INDEX_FILE: `${env.GIT_INDEX_FILE}.none` }, |
| 113 | maxOutputBytes: |
| 114 | CONVERSION_ATTRIBUTES.length * (Buffer.byteLength(stdin) + paths.length * 256), |
| 115 | outputMode: "error", |
| 116 | }); |
| 117 | // Records are "<path> NUL <attribute> NUL <value> NUL", one per attribute, in input order. |
| 118 | const fields = result.stdout.split("\0"); |
| 119 | return paths.map((entryPath, index) => { |
| 120 | const values: string[] = []; |
| 121 | for (let attribute = 0; attribute < CONVERSION_ATTRIBUTES.length; attribute++) { |
| 122 | const field = (index * CONVERSION_ATTRIBUTES.length + attribute) * 3; |
| 123 | if (fields[field] !== entryPath || fields[field + 2] === undefined) return undefined; |
| 124 | values.push(fields[field + 2]!); |
| 125 | } |
| 126 | return values.join("\0"); |
| 127 | }); |
| 128 | }); |
| 129 | |
| 130 | // Config that changes the blob or mode `git add` records for an unchanged file. |
| 131 | const readConfig = (cwd: string) => |
| 132 | execute({ |
| 133 | operation, |
| 134 | cwd, |
| 135 | args: [ |
| 136 | "config", |
| 137 | "--null", |
| 138 | "--get-regexp", |
| 139 | "^(core\\.(autocrlf|eol|filemode)|filter\\.(unspecified|unset)\\..*)$", |
| 140 | ], |
| 141 | allowNonZeroExit: true, |
| 142 | }).pipe(Effect.map((result) => `${result.exitCode}\0${result.stdout}`)); |
| 143 | |
| 144 | /** |
| 145 | * Finds directories whose .gitattributes may have changed while Git hashed: a present file |
| 146 | * touched since just before the capture, or a missing one in a directory touched since then. |
| 147 | * Reading attributes before and after staging cannot see a change that was undone in between. |
| 148 | */ |
| 149 | const changingAttributeDirectories = Effect.fnUntraced(function* ( |
| 150 | topLevel: string, |
| 151 | paths: ReadonlyArray<string>, |
| 152 | settledNs: bigint, |
| 153 | ) { |
| 154 | const directories = new Set<string>(paths.length === 0 ? [] : [""]); |
| 155 | for (const entryPath of paths) { |
| 156 | for ( |
| 157 | let end = entryPath.lastIndexOf("/"); |
| 158 | end > 0 && !directories.has(entryPath.slice(0, end + 1)); |
| 159 | end = entryPath.lastIndexOf("/", end - 1) |
| 160 | ) { |
| 161 | directories.add(entryPath.slice(0, end + 1)); |
| 162 | } |
| 163 | } |
| 164 | const list = [...directories]; |
| 165 | const [files, parents] = yield* Effect.all([ |
| 166 | lstatAll( |
| 167 | topLevel, |
| 168 | list.map((directory) => `${directory}.gitattributes`), |
| 169 | ), |
| 170 | lstatAll(topLevel, list), |
| 171 | ]); |
| 172 | const changing = new Set( |
| 173 | list.filter((_, index) => { |
| 174 | const changedNs = (files[index] ?? parents[index])?.changedNs; |
| 175 | return changedNs === undefined || changedNs >= settledNs; |
| 176 | }), |
| 177 | ); |
| 178 | return (entryPath: string) => { |
| 179 | if (changing.has("")) return true; |
| 180 | for ( |
| 181 | let end = entryPath.lastIndexOf("/"); |
| 182 | end > 0; |
| 183 | end = entryPath.lastIndexOf("/", end - 1) |
| 184 | ) { |
| 185 | if (changing.has(entryPath.slice(0, end + 1))) return true; |
| 186 | } |
| 187 | return false; |
| 188 | }; |
| 189 | }); |
| 190 | |
| 191 | /** |
| 192 | * Adds the still-valid blobs to the scratch index, flagged so `git add` does not read their |
| 193 | * files. The index must hold exactly HEAD's paths. Returns the blobs it added, and the |
| 194 | * conversion state of every file this capture may remember, read before any of them is hashed. |
| 195 | */ |
| 196 | const reuse = Effect.fn("GitVcsDriver.checkpoints.reuseUntrackedBlobs")( |
| 197 | function* (input: { |
| 198 | readonly cwd: string; |
| 199 | readonly topLevel: string; |
| 200 | readonly head: string; |
| 201 | readonly env: NodeJS.ProcessEnv; |
| 202 | readonly indexConfig: ReadonlyArray<string>; |
| 203 | }) { |
| 204 | const snapshot = snapshots.get(input.cwd); |
| 205 | // A capture that fails after this point hashes everything once more, then remembers again. |
| 206 | snapshots.delete(input.cwd); |
| 207 | if (snapshot === undefined || snapshot.topLevel !== input.topLevel) return NOTHING; |
| 208 | let blobs = snapshot.blobs; |
| 209 | if (snapshot.head !== input.head) { |
| 210 | // A path HEAD now tracks, or one below a path it changed, is no longer untracked. |
| 211 | const moved = yield* execute({ |
| 212 | operation, |
| 213 | cwd: input.cwd, |
| 214 | args: ["diff-tree", "-r", "-z", "--name-only", "--no-renames", snapshot.head, input.head], |
| 215 | maxOutputBytes: LIST_MAX_OUTPUT_BYTES, |
| 216 | outputMode: "error", |
| 217 | }); |
| 218 | const changed = new Set(moved.stdout.split("\0")); |
| 219 | blobs = blobs.filter((blob) => { |
| 220 | for (let end = blob.path.length; end > 0; end = blob.path.lastIndexOf("/", end - 1)) { |
| 221 | if (changed.has(blob.path.slice(0, end))) return false; |
| 222 | } |
| 223 | return true; |
| 224 | }); |
| 225 | } |
| 226 | const stats = yield* lstatAll( |
| 227 | input.topLevel, |
| 228 | blobs.map((blob) => blob.path), |
| 229 | ); |
| 230 | blobs = blobs.filter((blob, index) => stats[index]?.stat === blob.stat); |
| 231 | // Git stages a directory that became a repository as one gitlink, not as its files. |
| 232 | const directories = new Set<string>(); |
| 233 | for (const blob of blobs) { |
| 234 | for ( |
| 235 | let end = blob.path.lastIndexOf("/"); |
| 236 | end > 0 && !directories.has(blob.path.slice(0, end)); |
| 237 | end = blob.path.lastIndexOf("/", end - 1) |
| 238 | ) { |
| 239 | directories.add(blob.path.slice(0, end)); |
| 240 | } |
| 241 | } |
| 242 | const directoryList = [...directories]; |
| 243 | const repositories = yield* lstatAll( |
| 244 | input.topLevel, |
| 245 | directoryList.map((directory) => `${directory}/.git`), |
| 246 | ); |
| 247 | const nested = directoryList.filter((_, index) => repositories[index] !== undefined); |
| 248 | if (nested.length > 0) { |
| 249 | blobs = blobs.filter( |
| 250 | (blob) => !nested.some((directory) => blob.path.startsWith(`${directory}/`)), |
| 251 | ); |
| 252 | } |
| 253 | const paths = [...blobs.map((blob) => blob.path), ...snapshot.pending]; |
| 254 | if (paths.length === 0) return NOTHING; |
| 255 | const unchecked = Effect.succeed({ exitCode: 1, stdout: "" }); |
| 256 | const stdin = `${blobs.map((blob) => `${blob.path}\0`).join("")}`; |
| 257 | const [config, attributes, ignored, objects] = yield* Effect.all( |
| 258 | [ |
| 259 | readConfig(input.cwd), |
| 260 | readAttributes(input.topLevel, input.env, paths), |
| 261 | // Without --no-index this reports only untracked paths, which these are in this index. |
| 262 | blobs.length === 0 |
| 263 | ? unchecked |
| 264 | : execute({ |
| 265 | operation, |
| 266 | cwd: input.topLevel, |
| 267 | args: [...input.indexConfig, "check-ignore", "-z", "--stdin"], |
| 268 | stdin, |
| 269 | env: input.env, |
| 270 | allowNonZeroExit: true, |
| 271 | maxOutputBytes: Buffer.byteLength(stdin), |
| 272 | outputMode: "error", |
| 273 | }), |
| 274 | // Blobs of deleted checkpoints can be pruned while this process still remembers them. |
| 275 | blobs.length === 0 |
| 276 | ? unchecked |
| 277 | : execute({ |
| 278 | operation, |
| 279 | cwd: input.cwd, |
| 280 | args: ["cat-file", "--batch-check=%(objectname) %(objecttype)"], |
| 281 | stdin: `${[...new Set(blobs.map((blob) => blob.oid))].join("\n")}\n`, |
| 282 | maxOutputBytes: blobs.length * 128, |
| 283 | outputMode: "error", |
| 284 | }), |
| 285 | ], |
| 286 | { concurrency: "unbounded" }, |
| 287 | ); |
| 288 | const before: ConversionState = { |
| 289 | config, |
| 290 | attributes: new Map(paths.map((entryPath, index) => [entryPath, attributes[index]])), |
| 291 | }; |
| 292 | if (config !== snapshot.config || ignored.exitCode > 1) return { reused: [], before }; |
| 293 | const ignoredPaths = new Set(ignored.stdout.split("\0")); |
| 294 | const present = new Set(objects.stdout.split("\n")); |
| 295 | blobs = blobs.filter( |
| 296 | (blob, index) => |
| 297 | attributes[index] === blob.attributes && |
| 298 | !ignoredPaths.has(blob.path) && |
| 299 | present.has(`${blob.oid} blob`), |
| 300 | ); |
| 301 | if (blobs.length === 0) return { reused: [], before }; |
| 302 | |
| 303 | const indexFile = input.env.GIT_INDEX_FILE!; |
| 304 | const { mtime } = yield* fileSystem.stat(indexFile); |
| 305 | if (Option.isNone(mtime)) return { reused: [], before }; |
| 306 | yield* execute({ |
| 307 | operation, |
| 308 | cwd: input.cwd, |
| 309 | // core.ignorestat marks the new entries assume-unchanged, so git add trusts them as they are. |
| 310 | args: [ |
| 311 | ...input.indexConfig, |
| 312 | "-c", |
| 313 | "core.ignorestat=true", |
| 314 | "update-index", |
| 315 | "-z", |
| 316 | "--index-info", |
| 317 | ], |
| 318 | stdin: blobs.map((blob) => `${blob.mode} ${blob.oid}\t${blob.path}\0`).join(""), |
| 319 | env: input.env, |
| 320 | }); |
| 321 | // Tracked entries are racy against the index's old timestamp, not this rewrite's. |
| 322 | const indexTime = Math.floor(mtime.value.getTime() / 1000); |
| 323 | yield* fileSystem.utimes(indexFile, indexTime, indexTime); |
| 324 | return { reused: blobs, before }; |
| 325 | }, |
| 326 | Effect.orElseSucceed(() => NOTHING), |
| 327 | ); |
| 328 | |
| 329 | /** Records the untracked files the published checkpoint `tree` holds, for the next capture. */ |
| 330 | const remember = Effect.fn("GitVcsDriver.checkpoints.rememberUntrackedBlobs")( |
| 331 | function* (input: { |
| 332 | readonly cwd: string; |
| 333 | readonly topLevel: string; |
| 334 | readonly head: string; |
| 335 | readonly tree: string; |
| 336 | readonly env: NodeJS.ProcessEnv; |
| 337 | readonly startedAt: number; |
| 338 | readonly stagingMs: number; |
| 339 | readonly reused: ReadonlyArray<UntrackedBlob>; |
| 340 | readonly before: ConversionState | undefined; |
| 341 | }) { |
| 342 | if (input.reused.length === 0 && input.stagingMs < MIN_STAGING_MS) return; |
| 343 | const listed = yield* execute({ |
| 344 | operation, |
| 345 | cwd: input.cwd, |
| 346 | args: [ |
| 347 | "diff-tree", |
| 348 | "-r", |
| 349 | "-z", |
| 350 | "--raw", |
| 351 | "--no-abbrev", |
| 352 | "--no-renames", |
| 353 | "--diff-filter=AD", |
| 354 | input.head, |
| 355 | input.tree, |
| 356 | ], |
| 357 | maxOutputBytes: LIST_MAX_OUTPUT_BYTES, |
| 358 | outputMode: "error", |
| 359 | }); |
| 360 | const reused = new Map(input.reused.map((blob) => [blob.path, blob])); |
| 361 | const kept: UntrackedBlob[] = []; |
| 362 | const added: Array<{ path: string; mode: string; oid: string }> = []; |
| 363 | // Records are ":<old mode> <new mode> <old oid> <new oid> <status> NUL <path> NUL". |
| 364 | const fields = listed.stdout.split("\0"); |
| 365 | for (let field = 0; field + 1 < fields.length; field += 2) { |
| 366 | const [, mode = "", , oid = "", status] = fields[field]!.split(" "); |
| 367 | const entryPath = fields[field + 1]!; |
| 368 | if (status === "D") { |
| 369 | // Git may still apply a deleted .gitattributes from the index, depending on staging order. |
| 370 | if (entryPath === ".gitattributes" || entryPath.endsWith("/.gitattributes")) return; |
| 371 | continue; |
| 372 | } |
| 373 | // U+FFFD marks a name that is not UTF-8 and so cannot be addressed from here. |
| 374 | if ((mode !== "100644" && mode !== "100755") || entryPath.includes("\uFFFD")) continue; |
| 375 | const previous = reused.get(entryPath); |
| 376 | if (previous !== undefined && previous.oid === oid && previous.mode === mode) { |
| 377 | kept.push(previous); |
| 378 | } else { |
| 379 | added.push({ path: entryPath, mode, oid }); |
| 380 | } |
| 381 | } |
| 382 | const stats = yield* lstatAll( |
| 383 | input.topLevel, |
| 384 | added.map((entry) => entry.path), |
| 385 | ); |
| 386 | const settledNs = BigInt(input.startedAt - SETTLED_MS) * 1_000_000n; |
| 387 | const hashed = added.flatMap((entry, index) => { |
| 388 | const stat = stats[index]; |
| 389 | return stat !== undefined && stat.regular |
| 390 | ? [{ ...entry, size: stat.size, stat: stat.stat, settled: stat.changedNs < settledNs }] |
| 391 | : []; |
| 392 | }); |
| 393 | const largest = [...kept, ...hashed] |
| 394 | .sort((left, right) => (left.size < right.size ? 1 : left.size > right.size ? -1 : 0)) |
| 395 | .slice(0, MAX_BLOBS); |
| 396 | if (largest.reduce((total, blob) => total + blob.size, 0n) < MIN_TOTAL_BYTES) return; |
| 397 | |
| 398 | // Git hashed these under rules nobody recorded. Reading the rules now would pair the blob |
| 399 | // with whatever they have since become, so only a reading from before staging counts. |
| 400 | const before = input.before; |
| 401 | const candidates = largest.flatMap((blob) => |
| 402 | "settled" in blob && blob.settled && before?.attributes.get(blob.path) !== undefined |
| 403 | ? [blob] |
| 404 | : [], |
| 405 | ); |
| 406 | const candidatePaths = candidates.map((blob) => blob.path); |
| 407 | const [config, attributes, changing] = yield* Effect.all( |
| 408 | [ |
| 409 | readConfig(input.cwd), |
| 410 | readAttributes(input.topLevel, input.env, candidatePaths), |
| 411 | changingAttributeDirectories(input.topLevel, candidatePaths, settledNs), |
| 412 | ], |
| 413 | { concurrency: "unbounded" }, |
| 414 | ); |
| 415 | // check-attr prints a filter driver named "unspecified" or "unset" like no filter at all. |
| 416 | if (/\0filter\./.test(config)) return; |
| 417 | const sameConfig = before !== undefined && config === before.config; |
| 418 | const admitted = candidates.flatMap((blob, index): UntrackedBlob[] => { |
| 419 | const value = attributes[index]; |
| 420 | // A clean filter is a program whose output can depend on more than the file. |
| 421 | return sameConfig && |
| 422 | value !== undefined && |
| 423 | value === before.attributes.get(blob.path) && |
| 424 | /\0(?:unspecified|unset)$/.test(value) && |
| 425 | !changing(blob.path) |
| 426 | ? [ |
| 427 | { |
| 428 | path: blob.path, |
| 429 | mode: blob.mode, |
| 430 | oid: blob.oid, |
| 431 | size: blob.size, |
| 432 | stat: blob.stat, |
| 433 | attributes: value, |
| 434 | }, |
| 435 | ] |
| 436 | : []; |
| 437 | }); |
| 438 | // Reused blobs were checked against the earlier config; a different one now vouches for none. |
| 439 | const blobs = sameConfig |
| 440 | ? [...largest.flatMap((blob) => ("settled" in blob ? [] : [blob])), ...admitted] |
| 441 | : []; |
| 442 | const remembered = new Set(blobs.map((blob) => blob.path)); |
| 443 | snapshots.set(input.cwd, { |
| 444 | topLevel: input.topLevel, |
| 445 | head: input.head, |
| 446 | config, |
| 447 | blobs, |
| 448 | pending: largest.flatMap((blob) => (remembered.has(blob.path) ? [] : [blob.path])), |
| 449 | }); |
| 450 | if (snapshots.size > MAX_WORKSPACES) snapshots.delete(snapshots.keys().next().value!); |
| 451 | }, |
| 452 | (effect) => Effect.ignore(effect), |
| 453 | ); |
| 454 | |
| 455 | return { reuse, remember }; |
| 456 | }; |