@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 456 lines 17.7 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off - Comparing a file with its last capture needs bigint lstat, which FileSystem lacks.
2 import * as NodeFSP from "node:fs/promises";
3
4 import * as Effect from "effect/Effect";
5 import type * as FileSystem from "effect/FileSystem";
6 import * as Option from "effect/Option";
7
8 import type * as VcsDriver from "./VcsDriver.ts";
9 import * as VcsProcess from "./VcsProcess.ts";
10
11 /** An untracked file's blob from the last capture, with everything that decided its content. */
12 interface UntrackedBlob {
13 readonly path: string;
14 readonly mode: string;
15 readonly oid: string;
16 readonly size: bigint;
17 readonly stat: string;
18 readonly attributes: string;
19 }
20
21 interface Snapshot {
22 readonly topLevel: string;
23 readonly head: string;
24 readonly config: string;
25 readonly blobs: ReadonlyArray<UntrackedBlob>;
26 /** Files hashed without a reading of their conversion state from before staging. */
27 readonly pending: ReadonlyArray<string>;
28 }
29
30 /** The conversion state read before staging; a blob is remembered only if it reads the same after. */
31 interface ConversionState {
32 readonly config: string;
33 readonly attributes: ReadonlyMap<string, string | undefined>;
34 }
35
36 export const NOTHING: {
37 readonly reused: ReadonlyArray<UntrackedBlob>;
38 readonly before: ConversionState | undefined;
39 } = { reused: [], before: undefined };
40
41 // Every attribute convert_to_git reads; a blob is reused only while all of them are unchanged.
42 const CONVERSION_ATTRIBUTES = [
43 "text",
44 "eol",
45 "crlf",
46 "ident",
47 "working-tree-encoding",
48 "filter",
49 ] as const;
50 const MAX_BLOBS = 20_000;
51 // Below this, hashing the files costs less than the Git processes that check them.
52 const MIN_TOTAL_BYTES = 16n * 1024n * 1024n;
53 // A capture that staged faster than this and reused nothing has nothing worth another Git process.
54 const MIN_STAGING_MS = 100;
55 // A file must have been left alone for this long before the capture, so an edit that lands
56 // while Git hashes it cannot hide inside one timestamp tick (two seconds on FAT).
57 const SETTLED_MS = 3_000;
58 const MAX_WORKSPACES = 64;
59 const STAT_BATCH = 512;
60 const LIST_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
61
62 const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
63
64 const lstatAll = (root: string, paths: ReadonlyArray<string>) =>
65 Effect.promise(async () => {
66 const stats: Array<
67 { stat: string; size: bigint; changedNs: bigint; regular: boolean } | undefined
68 > = [];
69 for (let start = 0; start < paths.length; start += STAT_BATCH) {
70 const batch = await Promise.all(
71 paths.slice(start, start + STAT_BATCH).map((entryPath) =>
72 NodeFSP.lstat(`${root}/${entryPath}`, { bigint: true }).then(
73 (stat) => ({
74 stat: `${stat.dev}:${stat.ino}:${stat.mode}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`,
75 size: stat.size,
76 changedNs: stat.mtimeNs > stat.ctimeNs ? stat.mtimeNs : stat.ctimeNs,
77 regular: stat.isFile(),
78 }),
79 () => undefined,
80 ),
81 ),
82 );
83 stats.push(...batch);
84 }
85 return stats;
86 });
87
88 /**
89 * Lets a capture skip rehashing untracked files that have not changed since the last capture of
90 * the same workspace. A blob is reused only when a fresh `git add` would produce the same one:
91 * same file stat, same conversion attributes and config, still not ignored, still not tracked.
92 */
93 export const make = (
94 execute: VcsDriver.VcsDriver["Service"]["execute"],
95 fileSystem: FileSystem.FileSystem,
96 ) => {
97 const snapshots = new Map<string, Snapshot>();
98
99 // Worktree files only: an index copy of a deleted .gitattributes applies or not by staging order.
100 const readAttributes = Effect.fn("GitVcsDriver.checkpoints.readConversionAttributes")(function* (
101 topLevel: string,
102 env: NodeJS.ProcessEnv,
103 paths: ReadonlyArray<string>,
104 ) {
105 if (paths.length === 0) return [];
106 const stdin = `${paths.join("\0")}\0`;
107 const result = yield* execute({
108 operation,
109 cwd: topLevel,
110 args: ["check-attr", "-z", "--stdin", ...CONVERSION_ATTRIBUTES],
111 stdin,
112 env: { ...env, GIT_INDEX_FILE: `${env.GIT_INDEX_FILE}.none` },
113 maxOutputBytes:
114 CONVERSION_ATTRIBUTES.length * (Buffer.byteLength(stdin) + paths.length * 256),
115 outputMode: "error",
116 });
117 // Records are "<path> NUL <attribute> NUL <value> NUL", one per attribute, in input order.
118 const fields = result.stdout.split("\0");
119 return paths.map((entryPath, index) => {
120 const values: string[] = [];
121 for (let attribute = 0; attribute < CONVERSION_ATTRIBUTES.length; attribute++) {
122 const field = (index * CONVERSION_ATTRIBUTES.length + attribute) * 3;
123 if (fields[field] !== entryPath || fields[field + 2] === undefined) return undefined;
124 values.push(fields[field + 2]!);
125 }
126 return values.join("\0");
127 });
128 });
129
130 // Config that changes the blob or mode `git add` records for an unchanged file.
131 const readConfig = (cwd: string) =>
132 execute({
133 operation,
134 cwd,
135 args: [
136 "config",
137 "--null",
138 "--get-regexp",
139 "^(core\\.(autocrlf|eol|filemode)|filter\\.(unspecified|unset)\\..*)$",
140 ],
141 allowNonZeroExit: true,
142 }).pipe(Effect.map((result) => `${result.exitCode}\0${result.stdout}`));
143
144 /**
145 * Finds directories whose .gitattributes may have changed while Git hashed: a present file
146 * touched since just before the capture, or a missing one in a directory touched since then.
147 * Reading attributes before and after staging cannot see a change that was undone in between.
148 */
149 const changingAttributeDirectories = Effect.fnUntraced(function* (
150 topLevel: string,
151 paths: ReadonlyArray<string>,
152 settledNs: bigint,
153 ) {
154 const directories = new Set<string>(paths.length === 0 ? [] : [""]);
155 for (const entryPath of paths) {
156 for (
157 let end = entryPath.lastIndexOf("/");
158 end > 0 && !directories.has(entryPath.slice(0, end + 1));
159 end = entryPath.lastIndexOf("/", end - 1)
160 ) {
161 directories.add(entryPath.slice(0, end + 1));
162 }
163 }
164 const list = [...directories];
165 const [files, parents] = yield* Effect.all([
166 lstatAll(
167 topLevel,
168 list.map((directory) => `${directory}.gitattributes`),
169 ),
170 lstatAll(topLevel, list),
171 ]);
172 const changing = new Set(
173 list.filter((_, index) => {
174 const changedNs = (files[index] ?? parents[index])?.changedNs;
175 return changedNs === undefined || changedNs >= settledNs;
176 }),
177 );
178 return (entryPath: string) => {
179 if (changing.has("")) return true;
180 for (
181 let end = entryPath.lastIndexOf("/");
182 end > 0;
183 end = entryPath.lastIndexOf("/", end - 1)
184 ) {
185 if (changing.has(entryPath.slice(0, end + 1))) return true;
186 }
187 return false;
188 };
189 });
190
191 /**
192 * Adds the still-valid blobs to the scratch index, flagged so `git add` does not read their
193 * files. The index must hold exactly HEAD's paths. Returns the blobs it added, and the
194 * conversion state of every file this capture may remember, read before any of them is hashed.
195 */
196 const reuse = Effect.fn("GitVcsDriver.checkpoints.reuseUntrackedBlobs")(
197 function* (input: {
198 readonly cwd: string;
199 readonly topLevel: string;
200 readonly head: string;
201 readonly env: NodeJS.ProcessEnv;
202 readonly indexConfig: ReadonlyArray<string>;
203 }) {
204 const snapshot = snapshots.get(input.cwd);
205 // A capture that fails after this point hashes everything once more, then remembers again.
206 snapshots.delete(input.cwd);
207 if (snapshot === undefined || snapshot.topLevel !== input.topLevel) return NOTHING;
208 let blobs = snapshot.blobs;
209 if (snapshot.head !== input.head) {
210 // A path HEAD now tracks, or one below a path it changed, is no longer untracked.
211 const moved = yield* execute({
212 operation,
213 cwd: input.cwd,
214 args: ["diff-tree", "-r", "-z", "--name-only", "--no-renames", snapshot.head, input.head],
215 maxOutputBytes: LIST_MAX_OUTPUT_BYTES,
216 outputMode: "error",
217 });
218 const changed = new Set(moved.stdout.split("\0"));
219 blobs = blobs.filter((blob) => {
220 for (let end = blob.path.length; end > 0; end = blob.path.lastIndexOf("/", end - 1)) {
221 if (changed.has(blob.path.slice(0, end))) return false;
222 }
223 return true;
224 });
225 }
226 const stats = yield* lstatAll(
227 input.topLevel,
228 blobs.map((blob) => blob.path),
229 );
230 blobs = blobs.filter((blob, index) => stats[index]?.stat === blob.stat);
231 // Git stages a directory that became a repository as one gitlink, not as its files.
232 const directories = new Set<string>();
233 for (const blob of blobs) {
234 for (
235 let end = blob.path.lastIndexOf("/");
236 end > 0 && !directories.has(blob.path.slice(0, end));
237 end = blob.path.lastIndexOf("/", end - 1)
238 ) {
239 directories.add(blob.path.slice(0, end));
240 }
241 }
242 const directoryList = [...directories];
243 const repositories = yield* lstatAll(
244 input.topLevel,
245 directoryList.map((directory) => `${directory}/.git`),
246 );
247 const nested = directoryList.filter((_, index) => repositories[index] !== undefined);
248 if (nested.length > 0) {
249 blobs = blobs.filter(
250 (blob) => !nested.some((directory) => blob.path.startsWith(`${directory}/`)),
251 );
252 }
253 const paths = [...blobs.map((blob) => blob.path), ...snapshot.pending];
254 if (paths.length === 0) return NOTHING;
255 const unchecked = Effect.succeed({ exitCode: 1, stdout: "" });
256 const stdin = `${blobs.map((blob) => `${blob.path}\0`).join("")}`;
257 const [config, attributes, ignored, objects] = yield* Effect.all(
258 [
259 readConfig(input.cwd),
260 readAttributes(input.topLevel, input.env, paths),
261 // Without --no-index this reports only untracked paths, which these are in this index.
262 blobs.length === 0
263 ? unchecked
264 : execute({
265 operation,
266 cwd: input.topLevel,
267 args: [...input.indexConfig, "check-ignore", "-z", "--stdin"],
268 stdin,
269 env: input.env,
270 allowNonZeroExit: true,
271 maxOutputBytes: Buffer.byteLength(stdin),
272 outputMode: "error",
273 }),
274 // Blobs of deleted checkpoints can be pruned while this process still remembers them.
275 blobs.length === 0
276 ? unchecked
277 : execute({
278 operation,
279 cwd: input.cwd,
280 args: ["cat-file", "--batch-check=%(objectname) %(objecttype)"],
281 stdin: `${[...new Set(blobs.map((blob) => blob.oid))].join("\n")}\n`,
282 maxOutputBytes: blobs.length * 128,
283 outputMode: "error",
284 }),
285 ],
286 { concurrency: "unbounded" },
287 );
288 const before: ConversionState = {
289 config,
290 attributes: new Map(paths.map((entryPath, index) => [entryPath, attributes[index]])),
291 };
292 if (config !== snapshot.config || ignored.exitCode > 1) return { reused: [], before };
293 const ignoredPaths = new Set(ignored.stdout.split("\0"));
294 const present = new Set(objects.stdout.split("\n"));
295 blobs = blobs.filter(
296 (blob, index) =>
297 attributes[index] === blob.attributes &&
298 !ignoredPaths.has(blob.path) &&
299 present.has(`${blob.oid} blob`),
300 );
301 if (blobs.length === 0) return { reused: [], before };
302
303 const indexFile = input.env.GIT_INDEX_FILE!;
304 const { mtime } = yield* fileSystem.stat(indexFile);
305 if (Option.isNone(mtime)) return { reused: [], before };
306 yield* execute({
307 operation,
308 cwd: input.cwd,
309 // core.ignorestat marks the new entries assume-unchanged, so git add trusts them as they are.
310 args: [
311 ...input.indexConfig,
312 "-c",
313 "core.ignorestat=true",
314 "update-index",
315 "-z",
316 "--index-info",
317 ],
318 stdin: blobs.map((blob) => `${blob.mode} ${blob.oid}\t${blob.path}\0`).join(""),
319 env: input.env,
320 });
321 // Tracked entries are racy against the index's old timestamp, not this rewrite's.
322 const indexTime = Math.floor(mtime.value.getTime() / 1000);
323 yield* fileSystem.utimes(indexFile, indexTime, indexTime);
324 return { reused: blobs, before };
325 },
326 Effect.orElseSucceed(() => NOTHING),
327 );
328
329 /** Records the untracked files the published checkpoint `tree` holds, for the next capture. */
330 const remember = Effect.fn("GitVcsDriver.checkpoints.rememberUntrackedBlobs")(
331 function* (input: {
332 readonly cwd: string;
333 readonly topLevel: string;
334 readonly head: string;
335 readonly tree: string;
336 readonly env: NodeJS.ProcessEnv;
337 readonly startedAt: number;
338 readonly stagingMs: number;
339 readonly reused: ReadonlyArray<UntrackedBlob>;
340 readonly before: ConversionState | undefined;
341 }) {
342 if (input.reused.length === 0 && input.stagingMs < MIN_STAGING_MS) return;
343 const listed = yield* execute({
344 operation,
345 cwd: input.cwd,
346 args: [
347 "diff-tree",
348 "-r",
349 "-z",
350 "--raw",
351 "--no-abbrev",
352 "--no-renames",
353 "--diff-filter=AD",
354 input.head,
355 input.tree,
356 ],
357 maxOutputBytes: LIST_MAX_OUTPUT_BYTES,
358 outputMode: "error",
359 });
360 const reused = new Map(input.reused.map((blob) => [blob.path, blob]));
361 const kept: UntrackedBlob[] = [];
362 const added: Array<{ path: string; mode: string; oid: string }> = [];
363 // Records are ":<old mode> <new mode> <old oid> <new oid> <status> NUL <path> NUL".
364 const fields = listed.stdout.split("\0");
365 for (let field = 0; field + 1 < fields.length; field += 2) {
366 const [, mode = "", , oid = "", status] = fields[field]!.split(" ");
367 const entryPath = fields[field + 1]!;
368 if (status === "D") {
369 // Git may still apply a deleted .gitattributes from the index, depending on staging order.
370 if (entryPath === ".gitattributes" || entryPath.endsWith("/.gitattributes")) return;
371 continue;
372 }
373 // U+FFFD marks a name that is not UTF-8 and so cannot be addressed from here.
374 if ((mode !== "100644" && mode !== "100755") || entryPath.includes("\uFFFD")) continue;
375 const previous = reused.get(entryPath);
376 if (previous !== undefined && previous.oid === oid && previous.mode === mode) {
377 kept.push(previous);
378 } else {
379 added.push({ path: entryPath, mode, oid });
380 }
381 }
382 const stats = yield* lstatAll(
383 input.topLevel,
384 added.map((entry) => entry.path),
385 );
386 const settledNs = BigInt(input.startedAt - SETTLED_MS) * 1_000_000n;
387 const hashed = added.flatMap((entry, index) => {
388 const stat = stats[index];
389 return stat !== undefined && stat.regular
390 ? [{ ...entry, size: stat.size, stat: stat.stat, settled: stat.changedNs < settledNs }]
391 : [];
392 });
393 const largest = [...kept, ...hashed]
394 .sort((left, right) => (left.size < right.size ? 1 : left.size > right.size ? -1 : 0))
395 .slice(0, MAX_BLOBS);
396 if (largest.reduce((total, blob) => total + blob.size, 0n) < MIN_TOTAL_BYTES) return;
397
398 // Git hashed these under rules nobody recorded. Reading the rules now would pair the blob
399 // with whatever they have since become, so only a reading from before staging counts.
400 const before = input.before;
401 const candidates = largest.flatMap((blob) =>
402 "settled" in blob && blob.settled && before?.attributes.get(blob.path) !== undefined
403 ? [blob]
404 : [],
405 );
406 const candidatePaths = candidates.map((blob) => blob.path);
407 const [config, attributes, changing] = yield* Effect.all(
408 [
409 readConfig(input.cwd),
410 readAttributes(input.topLevel, input.env, candidatePaths),
411 changingAttributeDirectories(input.topLevel, candidatePaths, settledNs),
412 ],
413 { concurrency: "unbounded" },
414 );
415 // check-attr prints a filter driver named "unspecified" or "unset" like no filter at all.
416 if (/\0filter\./.test(config)) return;
417 const sameConfig = before !== undefined && config === before.config;
418 const admitted = candidates.flatMap((blob, index): UntrackedBlob[] => {
419 const value = attributes[index];
420 // A clean filter is a program whose output can depend on more than the file.
421 return sameConfig &&
422 value !== undefined &&
423 value === before.attributes.get(blob.path) &&
424 /\0(?:unspecified|unset)$/.test(value) &&
425 !changing(blob.path)
426 ? [
427 {
428 path: blob.path,
429 mode: blob.mode,
430 oid: blob.oid,
431 size: blob.size,
432 stat: blob.stat,
433 attributes: value,
434 },
435 ]
436 : [];
437 });
438 // Reused blobs were checked against the earlier config; a different one now vouches for none.
439 const blobs = sameConfig
440 ? [...largest.flatMap((blob) => ("settled" in blob ? [] : [blob])), ...admitted]
441 : [];
442 const remembered = new Set(blobs.map((blob) => blob.path));
443 snapshots.set(input.cwd, {
444 topLevel: input.topLevel,
445 head: input.head,
446 config,
447 blobs,
448 pending: largest.flatMap((blob) => (remembered.has(blob.path) ? [] : [blob.path])),
449 });
450 if (snapshots.size > MAX_WORKSPACES) snapshots.delete(snapshots.keys().next().value!);
451 },
452 (effect) => Effect.ignore(effect),
453 );
454
455 return { reuse, remember };
456 };