@devries-cameron20 / t3code-1 / commits / 7e69e880b6
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): project search starts working once a slow content index finishes building (#37138)

Julius Marminge committed Nov 15, 2026 at 05:00 UTC 7e69e880b630ceb372e22a41987543d997771269
2 files changed +451 -30
apps/server/src/workspace/WorkspaceSearchIndex.test.ts
+177
@@ -12,6 +12,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess";
12 import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
13 import * as Deferred from "effect/Deferred";
14 import * as Effect from "effect/Effect";
15 +import { ChildProcessSpawner } from "effect/process";
16 import * as Fiber from "effect/Fiber";
17 import * as FileSystem from "effect/FileSystem";
18 import * as Layer from "effect/Layer";
@@ -294,6 +295,182 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
295 ).pipe(Effect.provide(testLayer)),
296 );
297
298 +it.live("answers content searches with git grep while a slow content index builds", () =>
299 + Effect.gen(function* () {
300 + const fileSystem = yield* FileSystem.FileSystem;
301 + const path = yield* Path.Path;
302 + const vcsProcess = yield* VcsProcess.VcsProcess;
303 + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-content-warmup-" });
304 + yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd });
305 + yield* fileSystem.writeFileString(path.join(cwd, ".gitignore"), "ignored.txt\n");
306 + yield* fileSystem.makeDirectory(path.join(cwd, "src"));
307 + yield* fileSystem.writeFileString(
308 + path.join(cwd, "src/a.ts"),
309 + "const Needle = 1;\r\nneedles(a.b);\nnone\n",
310 + );
311 + yield* fileSystem.writeFileString(path.join(cwd, "ignored.txt"), "needle\n");
312 + // fff leaves out what `.ignore` excludes, so git results must leave it out too.
313 + yield* fileSystem.writeFileString(path.join(cwd, ".ignore"), "out/\n");
314 + yield* fileSystem.makeDirectory(path.join(cwd, "out/deep"), { recursive: true });
315 + yield* fileSystem.writeFileString(path.join(cwd, "out/deep/g.txt"), "needle\n");
316 +
317 + let isWarmupComplete = false;
318 + const grep = vi.fn(async () => ({
319 + ok: true as const,
320 + value: {
321 + items: [],
322 + totalMatched: 0,
323 + totalFilesSearched: 0,
324 + totalFiles: 0,
325 + filteredFileCount: 0,
326 + nextCursor: null,
327 + } satisfies GrepResult,
328 + }));
329 + const finder = {
330 + destroy: vi.fn(),
331 + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: false })),
332 + getScanProgressAsync: async () => ({
333 + ok: true as const,
334 + value: { isScanning: false, isWarmupComplete },
335 + }),
336 + watch: () => ({ ok: true as const, value: () => undefined }),
337 + grepAsync: grep,
338 + } as unknown as FileFinder;
339 + const noPcreFinder = {
340 + ...finder,
341 + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
342 + } as unknown as FileFinder;
343 + vi.spyOn(FileFinder, "create")
344 + .mockReturnValueOnce({ ok: true, value: finder })
345 + .mockReturnValueOnce({ ok: true, value: noPcreFinder })
346 + .mockReturnValueOnce({ ok: true, value: noPcreFinder })
347 + .mockReturnValueOnce({ ok: true, value: noPcreFinder });
348 +
349 + const searchIndex = yield* WorkspaceSearchIndex.make(cwd, "content");
350 + const search = (input: Partial<Parameters<typeof searchIndex.searchContents>[0]>) =>
351 + searchIndex.searchContents({
352 + query: "needle",
353 + limit: 10,
354 + caseSensitive: false,
355 + wholeWord: false,
356 + useRegex: false,
357 + ...input,
358 + });
359 +
360 + expect(yield* search({})).toEqual({
361 + matches: [
362 + {
363 + path: "src/a.ts",
364 + lineNumber: 1,
365 + lineContent: "const Needle = 1;",
366 + matchRanges: [{ start: 6, end: 12 }],
367 + },
368 + {
369 + path: "src/a.ts",
370 + lineNumber: 2,
371 + lineContent: "needles(a.b);",
372 + matchRanges: [{ start: 0, end: 6 }],
373 + },
374 + ],
375 + truncated: false,
376 + });
377 + expect((yield* search({ caseSensitive: true, wholeWord: true })).matches).toEqual([]);
378 + expect((yield* search({ query: "a.b" })).matches.map((match) => match.matchRanges)).toEqual([
379 + [{ start: 8, end: 11 }],
380 + ]);
381 + expect(
382 + (yield* search({ query: "ne+dles?", useRegex: true })).matches.map(
383 + (match) => match.matchRanges,
384 + ),
385 + ).toEqual([[{ start: 6, end: 12 }], [{ start: 0, end: 7 }]]);
386 + expect(
387 + (yield* search({ query: "[a-z]e", useRegex: true })).matches.map(
388 + (match) => match.matchRanges,
389 + ),
390 + ).toEqual([
391 + [{ start: 4, end: 6 }],
392 + [
393 + { start: 6, end: 8 },
394 + { start: 10, end: 12 },
395 + ],
396 + [
397 + { start: 0, end: 2 },
398 + { start: 4, end: 6 },
399 + ],
400 + [{ start: 2, end: 4 }],
401 + ]);
402 + const invalid = yield* search({ query: "a.b(", useRegex: true });
403 + expect(invalid.matches).toEqual([]);
404 + expect(invalid.regexFallbackError).toBeDefined();
405 + expect(yield* search({ limit: 1 })).toMatchObject({ truncated: true });
406 + expect(grep).not.toHaveBeenCalled();
407 +
408 + // When git cannot give regex results or their ranges, fff answers once ready instead.
409 + const failingGitIndex = (
410 + failingArgument: string,
411 + stderr: string,
412 + output: { readonly exitCode: number; readonly stdout: string } = {
413 + exitCode: 128,
414 + stdout: "",
415 + },
416 + ) =>
417 + WorkspaceSearchIndex.make(cwd, "content").pipe(
418 + Effect.provideService(
419 + VcsProcess.VcsProcess,
420 + VcsProcess.VcsProcess.of({
421 + run: (input) =>
422 + input.args.includes(failingArgument)
423 + ? Effect.succeed({
424 + exitCode: ChildProcessSpawner.ExitCode(output.exitCode),
425 + stdout: output.stdout,
426 + stderr,
427 + stdoutTruncated: output.exitCode === 0,
428 + stderrTruncated: false,
429 + })
430 + : vcsProcess.run(input),
431 + }),
432 + ),
433 + );
434 + const regexInput = {
435 + query: "ne+dles?",
436 + limit: 10,
437 + caseSensitive: false,
438 + wholeWord: true,
439 + useRegex: true,
440 + };
441 + const noPcreIndex = yield* failingGitIndex(
442 + "-P",
443 + "fatal: cannot use Perl-compatible regexes when not compiled with USE_LIBPCRE\n",
444 + );
445 + expect((yield* noPcreIndex.searchContents(regexInput)).regexFallbackError).toBeUndefined();
446 + expect(grep).toHaveBeenCalledTimes(1);
447 + const noRangesIndex = yield* failingGitIndex("-o", "fatal: unexpected failure\n");
448 + yield* noRangesIndex.searchContents(regexInput);
449 + expect(grep).toHaveBeenCalledTimes(2);
450 + // Ranges output cut off after line 1: line 2 is left out and the result says so.
451 + const cutRangesIndex = yield* failingGitIndex("-o", "", {
452 + exitCode: 0,
453 + stdout: "src/a.ts\u00001\u00007\u0000Needle\nsrc/a.ts\u00002",
454 + });
455 + expect(yield* cutRangesIndex.searchContents(regexInput)).toEqual({
456 + matches: [
457 + {
458 + path: "src/a.ts",
459 + lineNumber: 1,
460 + lineContent: "const Needle = 1;",
461 + matchRanges: [{ start: 6, end: 12 }],
462 + },
463 + ],
464 + truncated: true,
465 + });
466 +
467 + isWarmupComplete = true;
468 + yield* search({});
469 + expect(grep).toHaveBeenCalledTimes(3);
470 + expect(finder.destroy).not.toHaveBeenCalled();
471 + }).pipe(Effect.scoped, Effect.provide(testLayer)),
472 +);
473 +
474 function fakeSymlinkFinder(
475 directories: ReadonlyArray<string>,
476 options: {
apps/server/src/workspace/WorkspaceSearchIndex.ts
+274 -30
@@ -57,6 +57,9 @@ const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
57 const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
58 const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
59 const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
60 +const GIT_GREP_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
61 +// fff's lineContent cap; matches past it are reported without ranges.
62 +const CONTENT_LINE_MAX_LENGTH = 512;
63 const SYMLINK_SCAN_CONCURRENCY = 16;
64 // A scan no watcher can invalidate is reused this long, so typing does not rescan every folder.
65 const UNWATCHED_SYMLINK_SCAN_TTL_MS = 2_000;
@@ -294,6 +297,47 @@ function mapContentMatchRanges(
297 }));
298 }
299
300 +function escapeRegExp(input: string): string {
301 + return input.replace(/[\\^$.*+?()[\]{}|/]/g, "\\$&");
302 +}
303 +
304 +// Git before 2.55 reports wrong `--column` values for every match after a line's first, so
305 +// later matches are found by their text, searching on from the end of the previous one.
306 +function locateRegexMatches(
307 + line: string,
308 + matches: { readonly column: number; readonly texts: ReadonlyArray<string> } | undefined,
309 +): Array<readonly [number, number]> {
310 + if (!matches) return [];
311 + const lineBytes = Buffer.from(line);
312 + const ranges: Array<readonly [number, number]> = [];
313 + let start = matches.column - 1;
314 + for (const text of matches.texts) {
315 + const textBytes = Buffer.from(text);
316 + if (ranges.length > 0) start = lineBytes.indexOf(textBytes, ranges.at(-1)![1]);
317 + if (start < 0) break;
318 + ranges.push([start, start + textBytes.length]);
319 + }
320 + return ranges;
321 +}
322 +
323 +// `git grep -z` records: NUL-separated fields, one record per line; a truncated last line is dropped.
324 +function parseGitGrepRecords(
325 + stdout: string,
326 + fieldCount: number,
327 + truncated: boolean,
328 +): Array<Array<string>> {
329 + const lines = stdout.split("\n");
330 + if (truncated || lines.at(-1) === "") lines.pop();
331 + const records: Array<Array<string>> = [];
332 + for (const line of lines) {
333 + const fields = line.split("\0");
334 + if (fields.length < fieldCount) continue;
335 + // Only the last field (the line or match text) can contain NUL bytes.
336 + records.push([...fields.slice(0, fieldCount - 1), fields.slice(fieldCount - 1).join("\0")]);
337 + }
338 + return records;
339 +}
340 +
341 // Post-filter whole words: consuming boundary regexes swallow adjacent separators and widen ranges; \b misses punctuation edges.
342 function isWholeWordRange(
343 line: string,
@@ -576,26 +620,30 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
620 }).pipe(Effect.orDie),
621 );
622 let initialScanTimedOut = false;
623 + // The content index reads every file, which takes minutes where antivirus scans each open,
624 + // so content searches do not wait for it; git grep answers until it is ready.
625 + let contentIndexPending = variant === "content";
626 const vcsProcess = yield* VcsProcess.VcsProcess;
627 let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows);
581 - yield* waitForIndexReady(
582 - cwd,
583 - finder,
584 - ({ reason, cause }) =>
585 - new WorkspaceSearchIndexCreateFailed({
586 - cwd,
587 - reason,
588 - cause,
589 - }),
590 - ).pipe(
591 - // The scan keeps running on this finder; failing here would destroy it and cache the failure.
592 - Effect.catchTags({
593 - WorkspaceSearchIndexScanTimedOut: () =>
594 - Effect.sync(() => {
595 - initialScanTimedOut = true;
628 + if (variant === "paths") {
629 + yield* waitForIndexReady(
630 + cwd,
631 + finder,
632 + ({ reason, cause }) =>
633 + new WorkspaceSearchIndexCreateFailed({
634 + cwd,
635 + reason,
636 + cause,
637 }),
597 - }),
598 - );
638 + ).pipe(
639 + Effect.catchTags({
640 + WorkspaceSearchIndexScanTimedOut: () =>
641 + Effect.sync(() => {
642 + initialScanTimedOut = true;
643 + }),
644 + }),
645 + );
646 + }
647
648 // fff's isScanning takes the picker lock too, so progress is read off the JS thread.
649 const isScanning = Effect.promise(async () => {
@@ -608,10 +656,203 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
656 });
657 const hasIncompleteInitialScan = () => (initialScanTimedOut ? isScanning : Effect.succeed(false));
658
659 + const isContentIndexReady = Effect.fn("WorkspaceSearchIndex.isContentIndexReady")(function* () {
660 + if (!contentIndexPending) return true;
661 + const progress = yield* Effect.promise(() =>
662 + finder.getScanProgressAsync().catch(() => undefined),
663 + );
664 + if (progress?.ok && !progress.value.isScanning && progress.value.isWarmupComplete) {
665 + contentIndexPending = false;
666 + }
667 + return !contentIndexPending;
668 + });
669 +
670 + const runGitGrep = Effect.fn("WorkspaceSearchIndex.runGitGrep")(function* (
671 + pattern: string,
672 + options: {
673 + readonly regexMode: boolean;
674 + readonly ignoreCase: boolean;
675 + readonly maxMatchesPerFile: number;
676 + readonly onlyMatching: boolean;
677 + },
678 + ) {
679 + return yield* vcsProcess
680 + .run({
681 + operation: "WorkspaceSearchIndex.gitGrep",
682 + command: "git",
683 + args: [
684 + "-c",
685 + "core.quotePath=false",
686 + "-c",
687 + "core.fsmonitor=false",
688 + "grep",
689 + "--untracked",
690 + "--no-color",
691 + "-I",
692 + "-n",
693 + "-z",
694 + "-m",
695 + String(options.maxMatchesPerFile),
696 + ...(options.onlyMatching ? ["-o", "--column"] : []),
697 + ...(options.ignoreCase ? ["-i"] : []),
698 + options.regexMode ? "-P" : "-F",
699 + "-e",
700 + pattern,
701 + ],
702 + cwd,
703 + allowNonZeroExit: true,
704 + // Defender scanning fff's warmup reads slows git too; a late answer beats a failure.
705 + timeoutMs: 30_000,
706 + maxOutputBytes: GIT_GREP_MAX_OUTPUT_BYTES,
707 + outputMode: "truncate",
708 + appendTruncationMarker: false,
709 + })
710 + .pipe(Effect.option);
711 + });
712 +
713 + // fff leaves out what `.ignore` files exclude, which git cannot evaluate; the nearest
714 + // `.ignore` that matches a path or one of its folders decides, as in decideByIgnoreFiles.
715 + const makeIgnoreFileFilter = () => {
716 + const ignoreFiles = new Map<string, IgnoreFile | undefined>();
717 + const readIgnoreFile = Effect.fn("WorkspaceSearchIndex.readIgnoreFile")(function* (
718 + directory: string,
719 + ) {
720 + if (ignoreFiles.has(directory)) return ignoreFiles.get(directory);
721 + const contents = yield* fileSystem
722 + .readFileString(path.join(cwd, directory, ".ignore"))
723 + .pipe(Effect.option);
724 + const file = Option.isSome(contents) ? parseIgnoreFile(contents.value) : undefined;
725 + ignoreFiles.set(directory, file);
726 + return file;
727 + });
728 + return Effect.fn("WorkspaceSearchIndex.isExcludedByIgnoreFiles")(function* (
729 + relativePath: string,
730 + ) {
731 + const segments = relativePath.split("/");
732 + for (let depth = 1; depth <= segments.length; depth += 1) {
733 + const candidate = segments.slice(0, depth).join("/");
734 + for (let ancestor = depth - 1; ancestor >= 0; ancestor -= 1) {
735 + const directory = segments.slice(0, ancestor).join("/");
736 + const file = yield* readIgnoreFile(directory);
737 + const relative = directory ? candidate.slice(directory.length + 1) : candidate;
738 + const decision = file?.decide(relative, depth < segments.length);
739 + if (decision === "ignored") return true;
740 + if (decision === "included") break;
741 + }
742 + }
743 + return false;
744 + });
745 + };
746 +
747 + // fff's grep before its content index is ready opens every file itself, as slowly as the build.
748 + // None when git cannot search this folder (not a repository, git missing, no PCRE for a regex).
749 + const searchContentsWithGit = Effect.fn("WorkspaceSearchIndex.searchContentsWithGit")(function* (
750 + input: Omit<ProjectSearchContentsInput, "cwd">,
751 + ): Effect.fn.Return<Option.Option<ProjectSearchContentsResult>> {
752 + const options = {
753 + ignoreCase: !input.caseSensitive,
754 + maxMatchesPerFile: CONTENT_SEARCH_MAX_MATCHES_PER_FILE,
755 + onlyMatching: false,
756 + };
757 + let regexMode = input.useRegex;
758 + let regexFallbackError: string | undefined;
759 + let result = yield* runGitGrep(input.query, { ...options, regexMode });
760 + if (regexMode && Option.isSome(result) && result.value.exitCode === 128) {
761 + if (/USE_LIBPCRE|Perl-compatible/i.test(result.value.stderr)) return Option.none();
762 + // fff searches an invalid pattern as a literal and reports why.
763 + regexFallbackError = result.value.stderr.trim().split("\n")[0] || "Invalid regex";
764 + regexMode = false;
765 + result = yield* runGitGrep(input.query, { ...options, regexMode });
766 + }
767 + if (Option.isNone(result) || result.value.exitCode > 1) return Option.none();
768 +
769 + // Per line: the first match's byte column and every match's text, in order.
770 + const regexMatches = new Map<string, { readonly column: number; readonly texts: string[] }>();
771 + let rangesTruncated = false;
772 + if (regexMode) {
773 + // Ranges come from git too: running the user's pattern in JS could backtrack without bound.
774 + const ranges = yield* runGitGrep(input.query, { ...options, regexMode, onlyMatching: true });
775 + // Without ranges, whole-word filtering would drop every hit and report no matches.
776 + if (Option.isNone(ranges) || ranges.value.exitCode > 1) return Option.none();
777 + rangesTruncated = ranges.value.stdoutTruncated;
778 + if (ranges.value.exitCode === 0) {
779 + for (const record of parseGitGrepRecords(
780 + ranges.value.stdout,
781 + 4,
782 + ranges.value.stdoutTruncated,
783 + )) {
784 + const [relativePath, lineNumber, column, text] = record as [
785 + string,
786 + string,
787 + string,
788 + string,
789 + ];
790 + const key = `${relativePath}\0${lineNumber}`;
791 + const lineMatches = regexMatches.get(key);
792 + if (lineMatches) lineMatches.texts.push(text);
793 + else regexMatches.set(key, { column: Number(column), texts: [text] });
794 + }
795 + }
796 + }
797 + const literal = regexMode
798 + ? undefined
799 + : new RegExp(escapeRegExp(input.query), input.caseSensitive ? "gu" : "giu");
800 +
801 + const isExcludedByIgnoreFiles = makeIgnoreFileFilter();
802 + const matches: Array<ProjectSearchContentsResult["matches"][number]> = [];
803 + let rangesCutOff = false;
804 + for (const record of parseGitGrepRecords(
805 + result.value.stdout,
806 + 3,
807 + result.value.stdoutTruncated,
808 + )) {
809 + const [relativePath, lineNumber, rawLine] = record as [string, string, string];
810 + if (yield* isExcludedByIgnoreFiles(relativePath)) continue;
811 + const key = `${relativePath}\0${lineNumber}`;
812 + // Lines past the cut-off of a truncated ranges run have no ranges; stop there instead.
813 + if (rangesTruncated && !regexMatches.has(key)) {
814 + rangesCutOff = true;
815 + break;
816 + }
817 + const lineContent = (rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine).slice(
818 + 0,
819 + CONTENT_LINE_MAX_LENGTH,
820 + );
821 + const matchRanges = (
822 + literal
823 + ? Array.from(lineContent.matchAll(literal), (match) => ({
824 + start: match.index,
825 + end: match.index + match[0].length,
826 + }))
827 + : mapContentMatchRanges(
828 + lineContent,
829 + locateRegexMatches(lineContent, regexMatches.get(key)),
830 + )
831 + ).filter(
832 + (range) =>
833 + range.end <= lineContent.length &&
834 + (!input.wholeWord || isWholeWordRange(lineContent, range)),
835 + );
836 + if (input.wholeWord && matchRanges.length === 0) continue;
837 + matches.push({
838 + path: relativePath,
839 + lineNumber: Number(lineNumber),
840 + lineContent,
841 + matchRanges,
842 + });
843 + if (matches.length > input.limit) break;
844 + }
845 + return Option.some({
846 + matches: matches.slice(0, input.limit),
847 + truncated: matches.length > input.limit || result.value.stdoutTruncated || rangesCutOff,
848 + ...(regexFallbackError !== undefined ? { regexFallbackError } : {}),
849 + });
850 + });
851 +
852 const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* <A>(
853 query: string,
854 pageSize: number,
614 - operation: "directorySearch" | "fileSearch" | "getScanProgress" | "grep" | "mixedSearch",
855 + operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch",
856 execute: () => Promise<Result<A>>,
857 ): Effect.fn.Return<A, WorkspaceSearchIndexSearchFailed> {
858 // fff's searches wait on its picker lock, which its watcher can hold for minutes.
@@ -1038,20 +1279,23 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
1279 const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn(
1280 "WorkspaceSearchIndex.searchContents",
1281 )(function* (input) {
1041 - if (initialScanTimedOut) {
1042 - const progress = yield* runSearch(input.query, input.limit, "getScanProgress", () =>
1043 - finder.getScanProgressAsync(),
1282 + if (!(yield* isContentIndexReady())) {
1283 + const gitResult = yield* searchContentsWithGit(input);
1284 + if (Option.isSome(gitResult)) return gitResult.value;
1285 + yield* waitForIndexReady(
1286 + cwd,
1287 + finder,
1288 + ({ reason, cause }) =>
1289 + new WorkspaceSearchIndexSearchFailed({
1290 + cwd,
1291 + queryLength: input.query.length,
1292 + pageSize: input.limit,
1293 + reason,
1294 + cause,
1295 + }),
1296 );
1045 - // Files the first scan has not indexed yet would read as "No results found".
1046 - if (!progress.isWarmupComplete) {
1047 - return yield* new WorkspaceSearchIndexScanTimedOut({
1048 - cwd,
1049 - timeout: WORKSPACE_INDEX_SCAN_TIMEOUT,
1050 - });
1051 - }
1052 - initialScanTimedOut = false;
1297 + contentIndexPending = false;
1298 }
1054 -
1299 let { searchQuery, regexMode } = buildContentSearchQuery(input);
1300 const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS;
1301 // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing.