@devries-cameron20 / t3code-1 / commits / 39e269a37c
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): checkpoint capture no longer rehashes unchanged untracked files (#37482)

Julius Marminge committed Nov 15, 2026 at 21:28 UTC 39e269a37c13e9cef890804609046cfe5f3e7c4e
3 files changed +620
apps/server/src/vcs/GitVcsDriver.test.ts
+136
@@ -154,6 +154,15 @@ it.effect.each([
154 }).pipe(Effect.scoped, Effect.provide(layerGitContract)),
155 );
156
157 +interface UntrackedBlobChange {
158 + readonly git: Effect.Success<ReturnType<typeof makeCheckpointFixture>>["git"];
159 + readonly write: (
160 + relativePath: string,
161 + contents: string,
162 + ) => Effect.Effect<void, PlatformError.PlatformError>;
163 + readonly checkpointRef: CheckpointRef;
164 +}
165 +
166 const makeCheckpointFixture = Effect.fn("makeCheckpointFixture")(function* (
167 driver: Effect.Success<ReturnType<typeof GitVcsDriver.makeVcsDriverShape>>,
168 cwd: string,
@@ -529,6 +538,133 @@ it.effect("checkpoint capture ignores an inherited object directory", () =>
538 }).pipe(Effect.scoped, Effect.provide(layerGitContract)),
539 );
540
541 +it.effect.each([
542 + { name: "nothing changed", reuses: true, change: () => Effect.void },
543 + {
544 + name: "its conversion attributes changed while it was being captured",
545 + reuses: false,
546 + change: () => Effect.void,
547 + during: ({ write }: UntrackedBlobChange) =>
548 + Effect.orDie(write(".gitattributes", "*.txt text eol=lf\n")),
549 + },
550 + {
551 + name: "its conversion config changed while it was being captured",
552 + reuses: false,
553 + change: () => Effect.void,
554 + during: ({ git }: UntrackedBlobChange) =>
555 + Effect.orDie(Effect.asVoid(git(["config", "core.autocrlf", "true"]))),
556 + },
557 + {
558 + name: "its conversion attributes changed",
559 + reuses: false,
560 + change: ({ write }: UntrackedBlobChange) => write(".gitattributes", "*.txt text eol=lf\n"),
561 + },
562 + {
563 + name: "a clean filter named like an unset attribute now applies to it",
564 + reuses: false,
565 + change: ({ git, write }: UntrackedBlobChange) =>
566 + write(".git/info/attributes", "*.txt filter=unspecified\n").pipe(
567 + Effect.andThen(git(["config", "filter.unspecified.clean", "tr a-z A-Z"])),
568 + ),
569 + },
570 + {
571 + name: "its conversion config changed",
572 + reuses: false,
573 + change: ({ git }: UntrackedBlobChange) => git(["config", "core.autocrlf", "true"]),
574 + },
575 + {
576 + name: "it was rewritten at the same size",
577 + reuses: false,
578 + change: ({ write }: UntrackedBlobChange) =>
579 + write("nested/large.txt", "LINE\r\n".repeat(3 * 1024 * 1024)),
580 + },
581 + {
582 + name: "it became ignored",
583 + reuses: false,
584 + change: ({ write }: UntrackedBlobChange) => write(".gitignore", "nested/\n"),
585 + },
586 + {
587 + name: "HEAD now tracks it",
588 + reuses: false,
589 + change: ({ git }: UntrackedBlobChange) =>
590 + git(["add", "nested/large.txt"]).pipe(Effect.andThen(git(["commit", "-m", "track"]))),
591 + },
592 + {
593 + name: "its directory became a repository",
594 + reuses: false,
595 + change: ({ git }: UntrackedBlobChange) =>
596 + git(["init", "nested"]).pipe(
597 + Effect.andThen(
598 + git(
599 + ["-C", "nested", "-c", "user.name=Test", "-c", "user.email=test@test.com"].concat([
600 + "commit",
601 + "--allow-empty",
602 + "-m",
603 + "nested",
604 + ]),
605 + ),
606 + ),
607 + ),
608 + },
609 + {
610 + name: "its blob was pruned",
611 + reuses: false,
612 + change: ({ git, checkpointRef }: UntrackedBlobChange) =>
613 + git(["update-ref", "-d", checkpointRef]).pipe(Effect.andThen(git(["prune", "--expire=now"]))),
614 + },
615 +])(
616 + "checkpoint capture skips hashing an untracked file only while a fresh hash would match: $name",
617 + ({ reuses, change, during }) =>
618 + Effect.gen(function* () {
619 + const fs = yield* FileSystem.FileSystem;
620 + const path = yield* Path.Path;
621 + const liveProcess = yield* VcsProcess.VcsProcess;
622 + let reused = 0;
623 + let afterPublish: Effect.Effect<void> = Effect.void;
624 + const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
625 + Effect.provideService(VcsProcess.VcsProcess, {
626 + run: (input) => {
627 + if (input.args.includes("core.ignorestat=true")) reused++;
628 + // Only a capture whose staging took real time looks for files to remember.
629 + if (input.args.includes("add"))
630 + return liveProcess.run(input).pipe(Effect.tap(() => TestClock.adjust("1 second")));
631 + // The files are hashed and published by now, but not yet remembered.
632 + return input.args.includes("update-ref")
633 + ? liveProcess.run(input).pipe(Effect.tap(() => afterPublish))
634 + : liveProcess.run(input);
635 + },
636 + }),
637 + );
638 + // A driver that has never captured this workspace hashes every file, as a first capture does.
639 + const fresh = yield* GitVcsDriver.makeVcsDriverShape();
640 + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-untracked-blobs-" });
641 + const { git, checkpointRef } = yield* makeCheckpointFixture(driver, cwd);
642 + const write = (relativePath: string, contents: string) =>
643 + fs.writeFileString(path.join(cwd, relativePath), contents);
644 + yield* fs.makeDirectory(path.join(cwd, "nested"));
645 + yield* write("nested/large.txt", "line\r\n".repeat(3 * 1024 * 1024));
646 + // Only files left alone for a few seconds before a capture are remembered.
647 + yield* TestClock.adjust("36500 days");
648 + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
649 + // A file is remembered by the first capture that read its conversion rules before hashing it.
650 + if (during !== undefined) afterPublish = during({ git, write, checkpointRef });
651 + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
652 + afterPublish = Effect.void;
653 +
654 + yield* change({ git, write, checkpointRef });
655 + yield* driver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
656 + const freshRef = CheckpointRef.make("refs/t3/checkpoints/fresh");
657 + yield* fresh.checkpoints.captureCheckpoint({ cwd, checkpointRef: freshRef });
658 +
659 + assert.strictEqual(reused, reuses ? 1 : 0);
660 + assert.strictEqual(
661 + (yield* git(["rev-parse", `${checkpointRef}^{tree}`])).stdout,
662 + (yield* git(["rev-parse", `${freshRef}^{tree}`])).stdout,
663 + );
664 + yield* git(["fsck", "--connectivity-only"]);
665 + }).pipe(Effect.scoped, Effect.provide(layerGitContract)),
666 +);
667 +
668 it.effect.each(["sha1", "sha256"] as const)(
669 "checkpoint object isolation supports linked worktrees with $0 objects",
670 (format) =>
apps/server/src/vcs/GitVcsDriver.ts
+28
@@ -1,5 +1,6 @@
1 import * as NodeBuffer from "node:buffer";
2
3 +import * as Clock from "effect/Clock";
4 import * as Context from "effect/Context";
5 import * as SecureRandom from "effect/crypto/SecureRandom";
6 import * as DateTime from "effect/DateTime";
@@ -46,6 +47,7 @@ import {
47 splitNullSeparatedGitStdoutPaths,
48 } from "./GitVcsDriverCore.ts";
49 import * as CheckoutCloner from "./CheckoutCloner.ts";
50 +import * as CheckpointUntrackedBlobs from "./checkpointUntrackedBlobs.ts";
51 import * as VcsDriver from "./VcsDriver.ts";
52 import * as VcsProcess from "./VcsProcess.ts";
53
@@ -1066,6 +1068,8 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
1068 }),
1069 );
1070
1071 + const untrackedBlobs = CheckpointUntrackedBlobs.make(execute, fileSystem);
1072 +
1073 const checkpoints: VcsDriver.VcsCheckpointOps = {
1074 captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) {
1075 const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
@@ -1091,6 +1095,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
1095 lastMove !== undefined && lastMove.to === head
1096 ? `${lastMove.at} ${headMoves.filter((move) => move.at === lastMove.at).length}`
1097 : undefined;
1098 + const startedAt = yield* Clock.currentTimeMillis;
1099 const indexId = yield* random.uuidV4();
1100 const tempIndexPath = path.join(gitCommonDir, `t3-checkpoint-index-${indexId}`);
1101 const hostEnv = yield* HostProcess.Environment;
@@ -1379,6 +1384,16 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
1384 }
1385 }
1386
1387 + // Untracked files have no stat data in a new index, so git add would rehash every one.
1388 + const untracked =
1389 + topLevel === undefined || head === undefined || sparseCheckout || workspaceIgnored
1390 + ? undefined
1391 + : { cwd: input.cwd, topLevel, head, env: commitEnv };
1392 + const { reused, before } =
1393 + untracked === undefined
1394 + ? CheckpointUntrackedBlobs.NOTHING
1395 + : yield* untrackedBlobs.reuse({ ...untracked, indexConfig });
1396 +
1397 const stageFiles = (exclusions: ReadonlyArray<string>) =>
1398 execute({
1399 operation,
@@ -1400,6 +1415,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
1415 ],
1416 env: commitEnv,
1417 });
1418 + const stagingAt = yield* Clock.currentTimeMillis;
1419 yield* stageFiles([]).pipe(
1420 Effect.catchTags({
1421 VcsProcessExitError: (error) =>
@@ -1458,6 +1474,8 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
1474 }),
1475 );
1476
1477 + const stagingMs = (yield* Clock.currentTimeMillis) - stagingAt;
1478 +
1479 const writeTreeResult = yield* execute({
1480 operation,
1481 cwd: input.cwd,
@@ -1723,6 +1741,16 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
1741 cwd: input.cwd,
1742 args: [...durableWrite, "update-ref", input.checkpointRef, commitOid],
1743 });
1744 + if (untracked !== undefined) {
1745 + yield* untrackedBlobs.remember({
1746 + ...untracked,
1747 + tree: treeOid,
1748 + startedAt,
1749 + stagingMs,
1750 + reused,
1751 + before,
1752 + });
1753 + }
1754 }).pipe(Effect.ensuring(cleanupTempIndex));
1755
1756 // The tree records only each submodule's HEAD, so submodules keep their own ref.
apps/server/src/vcs/checkpointUntrackedBlobs.ts new
+456
@@ -0,0 +1,456 @@
1 +// @effect-diagnostics nodeBuiltinImport:off - Comparing a file with its last capture needs bigint lstat, which FileSystem lacks.
2 +import * as NodeFSP from "node:fs/promises";
3 +
4 +import * as Effect from "effect/Effect";
5 +import type * as FileSystem from "effect/FileSystem";
6 +import * as Option from "effect/Option";
7 +
8 +import type * as VcsDriver from "./VcsDriver.ts";
9 +import * as VcsProcess from "./VcsProcess.ts";
10 +
11 +/** An untracked file's blob from the last capture, with everything that decided its content. */
12 +interface UntrackedBlob {
13 + readonly path: string;
14 + readonly mode: string;
15 + readonly oid: string;
16 + readonly size: bigint;
17 + readonly stat: string;
18 + readonly attributes: string;
19 +}
20 +
21 +interface Snapshot {
22 + readonly topLevel: string;
23 + readonly head: string;
24 + readonly config: string;
25 + readonly blobs: ReadonlyArray<UntrackedBlob>;
26 + /** Files hashed without a reading of their conversion state from before staging. */
27 + readonly pending: ReadonlyArray<string>;
28 +}
29 +
30 +/** The conversion state read before staging; a blob is remembered only if it reads the same after. */
31 +interface ConversionState {
32 + readonly config: string;
33 + readonly attributes: ReadonlyMap<string, string | undefined>;
34 +}
35 +
36 +export const NOTHING: {
37 + readonly reused: ReadonlyArray<UntrackedBlob>;
38 + readonly before: ConversionState | undefined;
39 +} = { reused: [], before: undefined };
40 +
41 +// Every attribute convert_to_git reads; a blob is reused only while all of them are unchanged.
42 +const CONVERSION_ATTRIBUTES = [
43 + "text",
44 + "eol",
45 + "crlf",
46 + "ident",
47 + "working-tree-encoding",
48 + "filter",
49 +] as const;
50 +const MAX_BLOBS = 20_000;
51 +// Below this, hashing the files costs less than the Git processes that check them.
52 +const MIN_TOTAL_BYTES = 16n * 1024n * 1024n;
53 +// A capture that staged faster than this and reused nothing has nothing worth another Git process.
54 +const MIN_STAGING_MS = 100;
55 +// A file must have been left alone for this long before the capture, so an edit that lands
56 +// while Git hashes it cannot hide inside one timestamp tick (two seconds on FAT).
57 +const SETTLED_MS = 3_000;
58 +const MAX_WORKSPACES = 64;
59 +const STAT_BATCH = 512;
60 +const LIST_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
61 +
62 +const operation = VcsProcess.CHECKPOINT_CAPTURE_OPERATION;
63 +
64 +const lstatAll = (root: string, paths: ReadonlyArray<string>) =>
65 + Effect.promise(async () => {
66 + const stats: Array<
67 + { stat: string; size: bigint; changedNs: bigint; regular: boolean } | undefined
68 + > = [];
69 + for (let start = 0; start < paths.length; start += STAT_BATCH) {
70 + const batch = await Promise.all(
71 + paths.slice(start, start + STAT_BATCH).map((entryPath) =>
72 + NodeFSP.lstat(`${root}/${entryPath}`, { bigint: true }).then(
73 + (stat) => ({
74 + stat: `${stat.dev}:${stat.ino}:${stat.mode}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`,
75 + size: stat.size,
76 + changedNs: stat.mtimeNs > stat.ctimeNs ? stat.mtimeNs : stat.ctimeNs,
77 + regular: stat.isFile(),
78 + }),
79 + () => undefined,
80 + ),
81 + ),
82 + );
83 + stats.push(...batch);
84 + }
85 + return stats;
86 + });
87 +
88 +/**
89 + * Lets a capture skip rehashing untracked files that have not changed since the last capture of
90 + * the same workspace. A blob is reused only when a fresh `git add` would produce the same one:
91 + * same file stat, same conversion attributes and config, still not ignored, still not tracked.
92 + */
93 +export const make = (
94 + execute: VcsDriver.VcsDriver["Service"]["execute"],
95 + fileSystem: FileSystem.FileSystem,
96 +) => {
97 + const snapshots = new Map<string, Snapshot>();
98 +
99 + // Worktree files only: an index copy of a deleted .gitattributes applies or not by staging order.
100 + const readAttributes = Effect.fn("GitVcsDriver.checkpoints.readConversionAttributes")(function* (
101 + topLevel: string,
102 + env: NodeJS.ProcessEnv,
103 + paths: ReadonlyArray<string>,
104 + ) {
105 + if (paths.length === 0) return [];
106 + const stdin = `${paths.join("\0")}\0`;
107 + const result = yield* execute({
108 + operation,
109 + cwd: topLevel,
110 + args: ["check-attr", "-z", "--stdin", ...CONVERSION_ATTRIBUTES],
111 + stdin,
112 + env: { ...env, GIT_INDEX_FILE: `${env.GIT_INDEX_FILE}.none` },
113 + maxOutputBytes:
114 + CONVERSION_ATTRIBUTES.length * (Buffer.byteLength(stdin) + paths.length * 256),
115 + outputMode: "error",
116 + });
117 + // Records are "<path> NUL <attribute> NUL <value> NUL", one per attribute, in input order.
118 + const fields = result.stdout.split("\0");
119 + return paths.map((entryPath, index) => {
120 + const values: string[] = [];
121 + for (let attribute = 0; attribute < CONVERSION_ATTRIBUTES.length; attribute++) {
122 + const field = (index * CONVERSION_ATTRIBUTES.length + attribute) * 3;
123 + if (fields[field] !== entryPath || fields[field + 2] === undefined) return undefined;
124 + values.push(fields[field + 2]!);
125 + }
126 + return values.join("\0");
127 + });
128 + });
129 +
130 + // Config that changes the blob or mode `git add` records for an unchanged file.
131 + const readConfig = (cwd: string) =>
132 + execute({
133 + operation,
134 + cwd,
135 + args: [
136 + "config",
137 + "--null",
138 + "--get-regexp",
139 + "^(core\\.(autocrlf|eol|filemode)|filter\\.(unspecified|unset)\\..*)$",
140 + ],
141 + allowNonZeroExit: true,
142 + }).pipe(Effect.map((result) => `${result.exitCode}\0${result.stdout}`));
143 +
144 + /**
145 + * Finds directories whose .gitattributes may have changed while Git hashed: a present file
146 + * touched since just before the capture, or a missing one in a directory touched since then.
147 + * Reading attributes before and after staging cannot see a change that was undone in between.
148 + */
149 + const changingAttributeDirectories = Effect.fnUntraced(function* (
150 + topLevel: string,
151 + paths: ReadonlyArray<string>,
152 + settledNs: bigint,
153 + ) {
154 + const directories = new Set<string>(paths.length === 0 ? [] : [""]);
155 + for (const entryPath of paths) {
156 + for (
157 + let end = entryPath.lastIndexOf("/");
158 + end > 0 && !directories.has(entryPath.slice(0, end + 1));
159 + end = entryPath.lastIndexOf("/", end - 1)
160 + ) {
161 + directories.add(entryPath.slice(0, end + 1));
162 + }
163 + }
164 + const list = [...directories];
165 + const [files, parents] = yield* Effect.all([
166 + lstatAll(
167 + topLevel,
168 + list.map((directory) => `${directory}.gitattributes`),
169 + ),
170 + lstatAll(topLevel, list),
171 + ]);
172 + const changing = new Set(
173 + list.filter((_, index) => {
174 + const changedNs = (files[index] ?? parents[index])?.changedNs;
175 + return changedNs === undefined || changedNs >= settledNs;
176 + }),
177 + );
178 + return (entryPath: string) => {
179 + if (changing.has("")) return true;
180 + for (
181 + let end = entryPath.lastIndexOf("/");
182 + end > 0;
183 + end = entryPath.lastIndexOf("/", end - 1)
184 + ) {
185 + if (changing.has(entryPath.slice(0, end + 1))) return true;
186 + }
187 + return false;
188 + };
189 + });
190 +
191 + /**
192 + * Adds the still-valid blobs to the scratch index, flagged so `git add` does not read their
193 + * files. The index must hold exactly HEAD's paths. Returns the blobs it added, and the
194 + * conversion state of every file this capture may remember, read before any of them is hashed.
195 + */
196 + const reuse = Effect.fn("GitVcsDriver.checkpoints.reuseUntrackedBlobs")(
197 + function* (input: {
198 + readonly cwd: string;
199 + readonly topLevel: string;
200 + readonly head: string;
201 + readonly env: NodeJS.ProcessEnv;
202 + readonly indexConfig: ReadonlyArray<string>;
203 + }) {
204 + const snapshot = snapshots.get(input.cwd);
205 + // A capture that fails after this point hashes everything once more, then remembers again.
206 + snapshots.delete(input.cwd);
207 + if (snapshot === undefined || snapshot.topLevel !== input.topLevel) return NOTHING;
208 + let blobs = snapshot.blobs;
209 + if (snapshot.head !== input.head) {
210 + // A path HEAD now tracks, or one below a path it changed, is no longer untracked.
211 + const moved = yield* execute({
212 + operation,
213 + cwd: input.cwd,
214 + args: ["diff-tree", "-r", "-z", "--name-only", "--no-renames", snapshot.head, input.head],
215 + maxOutputBytes: LIST_MAX_OUTPUT_BYTES,
216 + outputMode: "error",
217 + });
218 + const changed = new Set(moved.stdout.split("\0"));
219 + blobs = blobs.filter((blob) => {
220 + for (let end = blob.path.length; end > 0; end = blob.path.lastIndexOf("/", end - 1)) {
221 + if (changed.has(blob.path.slice(0, end))) return false;
222 + }
223 + return true;
224 + });
225 + }
226 + const stats = yield* lstatAll(
227 + input.topLevel,
228 + blobs.map((blob) => blob.path),
229 + );
230 + blobs = blobs.filter((blob, index) => stats[index]?.stat === blob.stat);
231 + // Git stages a directory that became a repository as one gitlink, not as its files.
232 + const directories = new Set<string>();
233 + for (const blob of blobs) {
234 + for (
235 + let end = blob.path.lastIndexOf("/");
236 + end > 0 && !directories.has(blob.path.slice(0, end));
237 + end = blob.path.lastIndexOf("/", end - 1)
238 + ) {
239 + directories.add(blob.path.slice(0, end));
240 + }
241 + }
242 + const directoryList = [...directories];
243 + const repositories = yield* lstatAll(
244 + input.topLevel,
245 + directoryList.map((directory) => `${directory}/.git`),
246 + );
247 + const nested = directoryList.filter((_, index) => repositories[index] !== undefined);
248 + if (nested.length > 0) {
249 + blobs = blobs.filter(
250 + (blob) => !nested.some((directory) => blob.path.startsWith(`${directory}/`)),
251 + );
252 + }
253 + const paths = [...blobs.map((blob) => blob.path), ...snapshot.pending];
254 + if (paths.length === 0) return NOTHING;
255 + const unchecked = Effect.succeed({ exitCode: 1, stdout: "" });
256 + const stdin = `${blobs.map((blob) => `${blob.path}\0`).join("")}`;
257 + const [config, attributes, ignored, objects] = yield* Effect.all(
258 + [
259 + readConfig(input.cwd),
260 + readAttributes(input.topLevel, input.env, paths),
261 + // Without --no-index this reports only untracked paths, which these are in this index.
262 + blobs.length === 0
263 + ? unchecked
264 + : execute({
265 + operation,
266 + cwd: input.topLevel,
267 + args: [...input.indexConfig, "check-ignore", "-z", "--stdin"],
268 + stdin,
269 + env: input.env,
270 + allowNonZeroExit: true,
271 + maxOutputBytes: Buffer.byteLength(stdin),
272 + outputMode: "error",
273 + }),
274 + // Blobs of deleted checkpoints can be pruned while this process still remembers them.
275 + blobs.length === 0
276 + ? unchecked
277 + : execute({
278 + operation,
279 + cwd: input.cwd,
280 + args: ["cat-file", "--batch-check=%(objectname) %(objecttype)"],
281 + stdin: `${[...new Set(blobs.map((blob) => blob.oid))].join("\n")}\n`,
282 + maxOutputBytes: blobs.length * 128,
283 + outputMode: "error",
284 + }),
285 + ],
286 + { concurrency: "unbounded" },
287 + );
288 + const before: ConversionState = {
289 + config,
290 + attributes: new Map(paths.map((entryPath, index) => [entryPath, attributes[index]])),
291 + };
292 + if (config !== snapshot.config || ignored.exitCode > 1) return { reused: [], before };
293 + const ignoredPaths = new Set(ignored.stdout.split("\0"));
294 + const present = new Set(objects.stdout.split("\n"));
295 + blobs = blobs.filter(
296 + (blob, index) =>
297 + attributes[index] === blob.attributes &&
298 + !ignoredPaths.has(blob.path) &&
299 + present.has(`${blob.oid} blob`),
300 + );
301 + if (blobs.length === 0) return { reused: [], before };
302 +
303 + const indexFile = input.env.GIT_INDEX_FILE!;
304 + const { mtime } = yield* fileSystem.stat(indexFile);
305 + if (Option.isNone(mtime)) return { reused: [], before };
306 + yield* execute({
307 + operation,
308 + cwd: input.cwd,
309 + // core.ignorestat marks the new entries assume-unchanged, so git add trusts them as they are.
310 + args: [
311 + ...input.indexConfig,
312 + "-c",
313 + "core.ignorestat=true",
314 + "update-index",
315 + "-z",
316 + "--index-info",
317 + ],
318 + stdin: blobs.map((blob) => `${blob.mode} ${blob.oid}\t${blob.path}\0`).join(""),
319 + env: input.env,
320 + });
321 + // Tracked entries are racy against the index's old timestamp, not this rewrite's.
322 + const indexTime = Math.floor(mtime.value.getTime() / 1000);
323 + yield* fileSystem.utimes(indexFile, indexTime, indexTime);
324 + return { reused: blobs, before };
325 + },
326 + Effect.orElseSucceed(() => NOTHING),
327 + );
328 +
329 + /** Records the untracked files the published checkpoint `tree` holds, for the next capture. */
330 + const remember = Effect.fn("GitVcsDriver.checkpoints.rememberUntrackedBlobs")(
331 + function* (input: {
332 + readonly cwd: string;
333 + readonly topLevel: string;
334 + readonly head: string;
335 + readonly tree: string;
336 + readonly env: NodeJS.ProcessEnv;
337 + readonly startedAt: number;
338 + readonly stagingMs: number;
339 + readonly reused: ReadonlyArray<UntrackedBlob>;
340 + readonly before: ConversionState | undefined;
341 + }) {
342 + if (input.reused.length === 0 && input.stagingMs < MIN_STAGING_MS) return;
343 + const listed = yield* execute({
344 + operation,
345 + cwd: input.cwd,
346 + args: [
347 + "diff-tree",
348 + "-r",
349 + "-z",
350 + "--raw",
351 + "--no-abbrev",
352 + "--no-renames",
353 + "--diff-filter=AD",
354 + input.head,
355 + input.tree,
356 + ],
357 + maxOutputBytes: LIST_MAX_OUTPUT_BYTES,
358 + outputMode: "error",
359 + });
360 + const reused = new Map(input.reused.map((blob) => [blob.path, blob]));
361 + const kept: UntrackedBlob[] = [];
362 + const added: Array<{ path: string; mode: string; oid: string }> = [];
363 + // Records are ":<old mode> <new mode> <old oid> <new oid> <status> NUL <path> NUL".
364 + const fields = listed.stdout.split("\0");
365 + for (let field = 0; field + 1 < fields.length; field += 2) {
366 + const [, mode = "", , oid = "", status] = fields[field]!.split(" ");
367 + const entryPath = fields[field + 1]!;
368 + if (status === "D") {
369 + // Git may still apply a deleted .gitattributes from the index, depending on staging order.
370 + if (entryPath === ".gitattributes" || entryPath.endsWith("/.gitattributes")) return;
371 + continue;
372 + }
373 + // U+FFFD marks a name that is not UTF-8 and so cannot be addressed from here.
374 + if ((mode !== "100644" && mode !== "100755") || entryPath.includes("\uFFFD")) continue;
375 + const previous = reused.get(entryPath);
376 + if (previous !== undefined && previous.oid === oid && previous.mode === mode) {
377 + kept.push(previous);
378 + } else {
379 + added.push({ path: entryPath, mode, oid });
380 + }
381 + }
382 + const stats = yield* lstatAll(
383 + input.topLevel,
384 + added.map((entry) => entry.path),
385 + );
386 + const settledNs = BigInt(input.startedAt - SETTLED_MS) * 1_000_000n;
387 + const hashed = added.flatMap((entry, index) => {
388 + const stat = stats[index];
389 + return stat !== undefined && stat.regular
390 + ? [{ ...entry, size: stat.size, stat: stat.stat, settled: stat.changedNs < settledNs }]
391 + : [];
392 + });
393 + const largest = [...kept, ...hashed]
394 + .sort((left, right) => (left.size < right.size ? 1 : left.size > right.size ? -1 : 0))
395 + .slice(0, MAX_BLOBS);
396 + if (largest.reduce((total, blob) => total + blob.size, 0n) < MIN_TOTAL_BYTES) return;
397 +
398 + // Git hashed these under rules nobody recorded. Reading the rules now would pair the blob
399 + // with whatever they have since become, so only a reading from before staging counts.
400 + const before = input.before;
401 + const candidates = largest.flatMap((blob) =>
402 + "settled" in blob && blob.settled && before?.attributes.get(blob.path) !== undefined
403 + ? [blob]
404 + : [],
405 + );
406 + const candidatePaths = candidates.map((blob) => blob.path);
407 + const [config, attributes, changing] = yield* Effect.all(
408 + [
409 + readConfig(input.cwd),
410 + readAttributes(input.topLevel, input.env, candidatePaths),
411 + changingAttributeDirectories(input.topLevel, candidatePaths, settledNs),
412 + ],
413 + { concurrency: "unbounded" },
414 + );
415 + // check-attr prints a filter driver named "unspecified" or "unset" like no filter at all.
416 + if (/\0filter\./.test(config)) return;
417 + const sameConfig = before !== undefined && config === before.config;
418 + const admitted = candidates.flatMap((blob, index): UntrackedBlob[] => {
419 + const value = attributes[index];
420 + // A clean filter is a program whose output can depend on more than the file.
421 + return sameConfig &&
422 + value !== undefined &&
423 + value === before.attributes.get(blob.path) &&
424 + /\0(?:unspecified|unset)$/.test(value) &&
425 + !changing(blob.path)
426 + ? [
427 + {
428 + path: blob.path,
429 + mode: blob.mode,
430 + oid: blob.oid,
431 + size: blob.size,
432 + stat: blob.stat,
433 + attributes: value,
434 + },
435 + ]
436 + : [];
437 + });
438 + // Reused blobs were checked against the earlier config; a different one now vouches for none.
439 + const blobs = sameConfig
440 + ? [...largest.flatMap((blob) => ("settled" in blob ? [] : [blob])), ...admitted]
441 + : [];
442 + const remembered = new Set(blobs.map((blob) => blob.path));
443 + snapshots.set(input.cwd, {
444 + topLevel: input.topLevel,
445 + head: input.head,
446 + config,
447 + blobs,
448 + pending: largest.flatMap((blob) => (remembered.has(blob.path) ? [] : [blob.path])),
449 + });
450 + if (snapshots.size > MAX_WORKSPACES) snapshots.delete(snapshots.keys().next().value!);
451 + },
452 + (effect) => Effect.ignore(effect),
453 + );
454 +
455 + return { reuse, remember };
456 +};