@devries-cameron20 / t3code Mirror · read-only / commits / 302b28c4ec
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

fix(server): cloning a Git LFS repo no longer looks stalled while it downloads (#39963)

Julius Marminge committed Nov 20, 2026 at 17:56 UTC 302b28c4ecb12c4b6e5eba8e5ac41ccd0fad2bd8
4 files changed +148 -2
apps/server/src/project/ProjectCloneTracker.test.ts
+3
@@ -361,6 +361,9 @@ describe("parseGitCloneProgressLine", () => {
361 percent: 78,
362 detail: null,
363 });
364 + expect(
365 + parseGitCloneProgressLine("Filtering content: 66% (2/3), 11.92 MiB | 1009.00 KiB/s"),
366 + ).toEqual({ stage: "checkout", percent: 66, detail: "11.92 MiB | 1009.00 KiB/s" });
367 expect(parseGitCloneProgressLine("remote: Enumerating objects: 10, done.")).toEqual({
368 stage: "counting",
369 percent: null,
apps/server/src/project/gitCloneProgress.ts
+18
@@ -14,6 +14,7 @@ const STAGE_PREFIXES: ReadonlyArray<readonly [RegExp, ProjectCloneStage]> = [
14 [/^Resolving deltas/, "resolving"],
15 [/^Updating files/, "checkout"],
16 [/^Checking out files/, "checkout"],
17 + [/^Filtering content/, "checkout"],
18 ];
19
20 // Linear on hostile remote output: the tail is trimmed in code (optional `\s*`/`.*?` groups backtrack
@@ -41,3 +42,20 @@ export function parseGitCloneProgressLine(line: string): GitCloneProgressLine |
42 detail,
43 };
44 }
45 +
46 +/** Git counts filtered files only as each one lands, so this prefix marks its Git LFS phase. */
47 +export function isGitFilterProgressLine(line: string): boolean {
48 + return line.trimStart().startsWith("Filtering content");
49 +}
50 +
51 +/** Git prints nothing while a Git LFS object downloads; this reports the bytes of the unfinished ones. */
52 +export function gitLfsDownloadProgressLine(
53 + partialBytes: number,
54 + percent: number | null,
55 +): GitCloneProgressLine {
56 + const size =
57 + partialBytes >= 1024 ** 3
58 + ? `${(partialBytes / 1024 ** 3).toFixed(2)} GiB`
59 + : `${(partialBytes / 1024 ** 2).toFixed(1)} MiB`;
60 + return { stage: "checkout", percent, detail: `Downloading Git LFS objects: ${size} in progress` };
61 +}
apps/server/src/sourceControl/SourceControlRepositoryService.test.ts
+80
@@ -10,6 +10,7 @@ import * as FileSystem from "effect/FileSystem";
10 import * as Path from "effect/Path";
11 import * as Layer from "effect/Layer";
12 import * as PlatformError from "effect/PlatformError";
13 +import * as Queue from "effect/Queue";
14 import { ChildProcessSpawner } from "effect/process";
15 import * as TestClock from "effect/testing/TestClock";
16
@@ -20,6 +21,7 @@ import {
21 } from "@t3tools/contracts";
22
23 import * as ServerConfig from "../config.ts";
24 +import type { GitCloneProgressLine } from "../project/gitCloneProgress.ts";
25 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
26 import type * as SourceControlProvider from "@t3tools/source-control-core/server/SourceControlProvider";
27 import * as SourceControlProviderRegistry from "./SourceControlProviderRegistry.ts";
@@ -362,6 +364,84 @@ it.effect("reports clone progress from git's stderr and keeps its error text on
364 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
365 );
366
367 +it.effect("reports a Git LFS download as progress only while its partial files change", () =>
368 + Effect.gen(function* () {
369 + const fs = yield* FileSystem.FileSystem;
370 + const path = yield* Path.Path;
371 + const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-source-control-clone-lfs-" });
372 + const destinationPath = path.join(parent, "t3code");
373 + const incomplete = path.join(destinationPath, ".git", "lfs", "incomplete");
374 + const progress = yield* Queue.unbounded<GitCloneProgressLine>();
375 + const firstDownloading = yield* Deferred.make<void>();
376 + const firstLanded = yield* Deferred.make<void>();
377 + const secondDownloading = yield* Deferred.make<void>();
378 + const secondLanded = yield* Deferred.make<void>();
379 + const filtering = "Filtering content: 50% (1/2), 1.50 MiB | 1.00 MiB/s";
380 +
381 + const clone = yield* Effect.gen(function* () {
382 + const service = yield* SourceControlRepositoryService.SourceControlRepositoryService;
383 + return yield* service.cloneRepository(
384 + { remoteUrl: CLONE_URLS.url, destinationPath },
385 + { onProgress: (line) => Queue.offer(progress, line), timeoutMs: null },
386 + );
387 + }).pipe(
388 + Effect.provide(
389 + layer({
390 + git: {
391 + execute: (input) =>
392 + Effect.gen(function* () {
393 + yield* (
394 + input.progress?.onStderrLine?.("Updating files: 100% (3/3), done.") ?? Effect.void
395 + );
396 + yield* fs.makeDirectory(incomplete, { recursive: true });
397 + yield* fs.writeFile(path.join(incomplete, "a.part"), new Uint8Array(1536 * 1024));
398 + yield* Deferred.succeed(firstDownloading, undefined);
399 + yield* Deferred.await(firstLanded);
400 + yield* fs.remove(path.join(incomplete, "a.part"));
401 + yield* fs.writeFile(path.join(incomplete, "b.part"), new Uint8Array(512 * 1024));
402 + yield* input.progress?.onStderrLine?.(filtering) ?? Effect.void;
403 + yield* Deferred.succeed(secondDownloading, undefined);
404 + yield* Deferred.await(secondLanded);
405 + yield* fs.remove(path.join(incomplete, "b.part"));
406 + return processOutput();
407 + }).pipe(Effect.orDie),
408 + },
409 + }),
410 + ),
411 + Effect.forkChild,
412 + );
413 +
414 + yield* Deferred.await(firstDownloading);
415 + yield* TestClock.adjust("2 seconds");
416 + assert.deepStrictEqual(yield* Queue.takeN(progress, 2), [
417 + { stage: "checkout", percent: 100, detail: null },
418 + {
419 + stage: "checkout",
420 + percent: null,
421 + detail: "Downloading Git LFS objects: 1.5 MiB in progress",
422 + },
423 + ]);
424 +
425 + yield* Deferred.succeed(firstLanded, undefined);
426 + yield* Deferred.await(secondDownloading);
427 + yield* TestClock.adjust("2 seconds");
428 + assert.deepStrictEqual(yield* Queue.takeN(progress, 2), [
429 + { stage: "checkout", percent: 50, detail: "1.50 MiB | 1.00 MiB/s" },
430 + {
431 + stage: "checkout",
432 + percent: 50,
433 + detail: "Downloading Git LFS objects: 0.5 MiB in progress",
434 + },
435 + ]);
436 +
437 + // The second object stops growing: a transfer that really stalled must stay silent.
438 + yield* TestClock.adjust("2 seconds");
439 + yield* Deferred.succeed(secondLanded, undefined);
440 + yield* Fiber.join(clone);
441 + assert.strictEqual(yield* Queue.size(progress), 0);
442 + }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
443 +);
444 +
445 it.effect("says a clone that ran out of time timed out, not git's last remote line", () =>
446 Effect.gen(function* () {
447 const fs = yield* FileSystem.FileSystem;
apps/server/src/sourceControl/SourceControlRepositoryService.ts
+47 -2
@@ -27,6 +27,8 @@ import * as ServerConfig from "../config.ts";
27 import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion";
28 import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit";
29 import {
30 + gitLfsDownloadProgressLine,
31 + isGitFilterProgressLine,
32 parseGitCloneProgressLine,
33 type GitCloneProgressLine,
34 } from "../project/gitCloneProgress.ts";
@@ -83,6 +85,8 @@ const CLONE_ENV = {
85 // English progress without overriding LC_CTYPE, which a global post-checkout hook inherits.
86 LANGUAGE: "C",
87 } satisfies NodeJS.ProcessEnv;
88 +// Well inside the tracker's stall window, at about the pace git itself reports.
89 +const LFS_DOWNLOAD_POLL_INTERVAL = Duration.seconds(2);
90
91 function mapRepositoryError(operation: string, provider: SourceControlProviderKind) {
92 return Effect.mapError((cause: unknown) =>
@@ -328,6 +332,36 @@ export const make = Effect.gen(function* () {
332 } satisfies SourceControlPreparedClone;
333 });
334
335 + // Git and git-lfs print nothing while one object downloads, so a healthy transfer reads as a
336 + // stall. Its partial files growing are the only sign of life a plain clone gives.
337 + const watchLfsDownload = (
338 + destinationPath: string,
339 + onChange: (partialBytes: number) => Effect.Effect<void>,
340 + ) => {
341 + const incomplete = path.join(destinationPath, ".git", "lfs", "incomplete");
342 + const partialBytes = fileSystem.readDirectory(incomplete).pipe(
343 + Effect.flatMap(
344 + Effect.forEach((name) =>
345 + fileSystem
346 + .stat(path.join(incomplete, name))
347 + .pipe(Effect.map((info) => Number(info.size))),
348 + ),
349 + ),
350 + Effect.map((sizes) => sizes.reduce((total, size) => total + size, 0)),
351 + // No directory means no LFS download; a part that landed mid-read counts on the next poll.
352 + Effect.orElseSucceed(() => 0),
353 + );
354 + return Effect.gen(function* () {
355 + let previous = 0;
356 + while (true) {
357 + yield* Effect.sleep(LFS_DOWNLOAD_POLL_INTERVAL);
358 + const current = yield* partialBytes;
359 + if (current > 0 && current !== previous) yield* onChange(current);
360 + previous = current;
361 + }
362 + });
363 + };
364 +
365 const runClone = Effect.fn("SourceControlRepositoryService.runClone")(function* (
366 input: SourceControlCloneRepositoryInput,
367 destinationPath: string,
@@ -337,8 +371,10 @@ export const make = Effect.gen(function* () {
371 const onProgress = options?.onProgress;
372 // Git mixes progress and diagnostics on stderr; retain non-progress tail lines for failure details.
373 const stderrTail: Array<string> = [];
374 + let filterPercent: number | null = null;
375 const onStderrLine = (line: string) => {
376 const parsed = parseGitCloneProgressLine(line);
377 + if (parsed && isGitFilterProgressLine(line)) filterPercent = parsed.percent;
378 if (parsed) return onProgress ? onProgress(parsed) : Effect.void;
379 return Effect.sync(() => {
380 const trimmed = line.trim();
@@ -380,9 +416,18 @@ export const make = Effect.gen(function* () {
416 }),
417 ),
418 );
383 - if (timeoutMs === null) return yield* clone;
419 + const watchedClone = onProgress
420 + ? clone.pipe(
421 + Effect.raceFirst(
422 + watchLfsDownload(destinationPath, (partialBytes) =>
423 + onProgress(gitLfsDownloadProgressLine(partialBytes, filterPercent)),
424 + ),
425 + ),
426 + )
427 + : clone;
428 + if (timeoutMs === null) return yield* watchedClone;
429 // Interrupting the clone closes the scope that owns git, which kills it.
385 - return yield* clone.pipe(
430 + return yield* watchedClone.pipe(
431 Effect.timeoutOrElse({
432 duration: timeoutMs,
433 orElse: () =>