@devries-cameron20 / t3code Mirror · read-only / commits / 3de49baa70
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 18:30 UTC

fix(server): workspace search leaves out files Git ignores outside .gitignore (#35673)

Julius Marminge committed Nov 12, 2026 at 15:16 UTC 3de49baa7065b6e32ad9d97505aace51fda6ede4
3 files changed +169 -17
apps/server/src/workspace/WorkspaceEntries.test.ts
+9
@@ -493,6 +493,11 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
493 yield* writeTextFile(cwd, "ignored.txt", "ignore me");
494 yield* writeTextFile(cwd, ".convex/local-storage/data.json", "{}");
495 yield* writeTextFile(cwd, "convex/UOoS-l/convex_local_storage/modules/data.json", "{}");
496 + yield* writeTextFile(cwd, ".git/info/exclude", "info-excluded.txt\n");
497 + yield* writeTextFile(cwd, "info-excluded.txt", "ignore me");
498 + yield* writeTextFile(cwd, ".git/configured-excludes", "configured-excluded.txt\n");
499 + yield* git(cwd, ["config", "core.excludesFile", ".git/configured-excludes"]);
500 + yield* writeTextFile(cwd, "configured-excluded.txt", "ignore me");
501
502 const result = yield* searchWorkspaceEntries({ cwd, query: "", limit: 100 });
503 const paths = result.entries.map((entry) => entry.path);
@@ -500,6 +505,8 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
505 expect(paths).toContain("src");
506 expect(paths).toContain("src/keep.ts");
507 expect(paths).not.toContain("ignored.txt");
508 + expect(paths).not.toContain("info-excluded.txt");
509 + expect(paths).not.toContain("configured-excluded.txt");
510 expect(paths.some((entryPath) => entryPath.startsWith(".convex/"))).toBe(false);
511 expect(paths.some((entryPath) => entryPath.startsWith("convex/"))).toBe(false);
512 }),
@@ -591,6 +598,8 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
598 yield* writeTextFile(cwd, ".gitignore", "ignored.txt\n");
599 yield* writeTextFile(cwd, "src/keep.ts", "square\nSquare\n");
600 yield* writeTextFile(cwd, "ignored.txt", "Square\n");
601 + yield* writeTextFile(cwd, ".git/info/exclude", "info-excluded.txt\n");
602 + yield* writeTextFile(cwd, "info-excluded.txt", "Square\n");
603
604 const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
605 const result = yield* workspaceEntries.searchContents({
apps/server/src/workspace/WorkspaceSearchIndex.test.ts
+49 -3
@@ -10,11 +10,15 @@ import { afterEach, expect, it } from "@effect/vitest";
10 import * as HostProcess from "@t3tools/shared/HostProcess";
11 import * as Effect from "effect/Effect";
12 import * as FileSystem from "effect/FileSystem";
13 +import * as Layer from "effect/Layer";
14 import * as Path from "effect/Path";
15 import { vi } from "vite-plus/test";
16
17 +import * as VcsProcess from "../vcs/VcsProcess.ts";
18 import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts";
19
20 +const testLayer = VcsProcess.layer.pipe(Layer.provideMerge(NodeServices.layer));
21 +
22 afterEach(() => {
23 vi.restoreAllMocks();
24 });
@@ -64,7 +68,49 @@ it.effect("filters image searches before applying the result limit", () =>
68 expect(fileSearch).toHaveBeenCalledTimes(2);
69 expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 });
70 }),
67 - ),
71 + ).pipe(Effect.provide(testLayer)),
72 +);
73 +
74 +it.effect("fetches past a first page of Git-ignored results", () =>
75 + Effect.scoped(
76 + Effect.gen(function* () {
77 + const fs = yield* FileSystem.FileSystem;
78 + const path = yield* Path.Path;
79 + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-search-ignored-" });
80 + const vcsProcess = yield* VcsProcess.VcsProcess;
81 + yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd });
82 + yield* fs.writeFileString(path.join(cwd, ".git", "info", "exclude"), "hidden/\n");
83 + yield* fs.makeDirectory(path.join(cwd, "hidden"));
84 + yield* fs.writeFileString(path.join(cwd, "hidden", "a.ts"), "");
85 +
86 + const items = [
87 + fileItem("hidden/a.ts"),
88 + fileItem("hidden/b.ts"),
89 + fileItem("hidden/c.ts"),
90 + fileItem("src/a.ts"),
91 + ];
92 + const fileSearch = vi.fn((_query: string, options?: { pageSize?: number }) => ({
93 + ok: true as const,
94 + value: {
95 + items: items.slice(0, options?.pageSize),
96 + scores: [],
97 + totalMatched: items.length,
98 + totalFiles: items.length,
99 + },
100 + }));
101 + const finder = {
102 + destroy: vi.fn(),
103 + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
104 + fileSearch,
105 + } as unknown as FileFinder;
106 + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
107 +
108 + const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
109 + const result = yield* searchIndex.search("a", 1, "file", false);
110 +
111 + expect(result).toEqual({ entries: [{ kind: "file", path: "src/a.ts" }], truncated: false });
112 + }),
113 + ).pipe(Effect.provide(testLayer)),
114 );
115
116 it.effect("continues whole-word searches after a filtered grep page", () =>
@@ -141,7 +187,7 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
187 expect(grep).toHaveBeenCalledTimes(2);
188 expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor);
189 }),
144 - ),
190 + ).pipe(Effect.provide(testLayer)),
191 );
192
193 // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there.
@@ -172,5 +218,5 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
218
219 expect(result).toBe("WorkspaceSearchIndexCreateFailed");
220 }),
175 - ).pipe(Effect.provide(NodeServices.layer)),
221 + ).pipe(Effect.provide(testLayer)),
222 );
apps/server/src/workspace/WorkspaceSearchIndex.ts
+111 -14
@@ -2,14 +2,11 @@ import * as NodeModule from "node:module";
2
3 import type {
4 DirItem,
5 - DirSearchResult,
5 FileItem,
6 FileFinder as FileFinderType,
7 GrepCursor,
8 MixedItem,
10 - MixedSearchResult,
9 Result,
12 - SearchResult,
10 } from "@ff-labs/fff-node";
11 import * as Context from "effect/Context";
12 import * as Effect from "effect/Effect";
@@ -27,6 +24,8 @@ import type {
24 } from "@t3tools/contracts";
25 import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
26
27 +import * as VcsProcess from "../vcs/VcsProcess.ts";
28 +
29 // fff-node's native library stays external; single-executables require disk packages instead of importing them.
30 const requireForFff = NodeModule.createRequire(import.meta.url);
31 const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node");
@@ -156,8 +155,10 @@ function toDirectoryEntry(item: DirItem): ProjectEntry | null {
155 return normalizedPath ? { path: normalizedPath, kind: "directory" } : null;
156 }
157
158 +type SearchPage<I> = { readonly items: ReadonlyArray<I>; readonly totalMatched: number };
159 +
160 function mapFileSearchResult(
160 - result: SearchResult,
161 + result: SearchPage<FileItem>,
162 limit: number,
163 imageOnly = false,
164 ): ProjectSearchEntriesResult {
@@ -172,7 +173,7 @@ function mapFileSearchResult(
173 }
174
175 function mapDirectorySearchResult(
175 - result: DirSearchResult,
176 + result: SearchPage<DirItem>,
177 limit: number,
178 ): ProjectSearchEntriesResult {
179 const entries = result.items.flatMap((item) => {
@@ -187,7 +188,7 @@ function mapDirectorySearchResult(
188 }
189
190 function mapMixedSearchResult(
190 - result: MixedSearchResult,
191 + result: SearchPage<MixedItem>,
192 limit: number,
193 ): { readonly entries: ProjectEntry[]; readonly truncated: boolean } {
194 const entries: ProjectEntry[] = [];
@@ -284,6 +285,54 @@ function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEn
285 return [...entryByPath.values()];
286 }
287
288 +// fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file.
289 +const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* (
290 + vcsProcess: VcsProcess.VcsProcess["Service"],
291 + cwd: string,
292 +) {
293 + const result = yield* vcsProcess
294 + .run({
295 + operation: "WorkspaceSearchIndex.listGitIgnoredPaths",
296 + command: "git",
297 + args: [
298 + "-c",
299 + "core.fsmonitor=false",
300 + "ls-files",
301 + "--others",
302 + "--ignored",
303 + "--exclude-standard",
304 + "--directory",
305 + "-z",
306 + ],
307 + cwd,
308 + allowNonZeroExit: true,
309 + timeoutMs: 10_000,
310 + maxOutputBytes: 16 * 1024 * 1024,
311 + })
312 + .pipe(Effect.orElseSucceed(() => undefined));
313 + if (!result || result.exitCode !== 0 || result.stdoutTruncated) return new Set<string>();
314 + return new Set(
315 + result.stdout
316 + .split("\0")
317 + .filter((entry) => entry.length > 0)
318 + .map((entry) => trimDirectorySeparator(toPosixPath(entry))),
319 + );
320 +});
321 +
322 +function mixedItemPath(item: MixedItem): string {
323 + return item.item.relativePath;
324 +}
325 +
326 +function isIgnoredPath(ignoredPaths: ReadonlySet<string>, relativePath: string): boolean {
327 + if (ignoredPaths.size === 0) return false;
328 + let candidate: string | undefined = trimDirectorySeparator(toPosixPath(relativePath));
329 + while (candidate) {
330 + if (ignoredPaths.has(candidate)) return true;
331 + candidate = parentPathOf(candidate);
332 + }
333 + return false;
334 +}
335 +
336 const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (
337 cwd: string,
338 variant: WorkspaceSearchIndexVariant,
@@ -346,6 +395,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
395 }).pipe(Effect.orDie),
396 );
397 let initialScanTimedOut = false;
398 + const vcsProcess = yield* VcsProcess.VcsProcess;
399 + let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd);
400 yield* waitForIndexReady(
401 cwd,
402 finder,
@@ -396,6 +447,34 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
447 return result.value;
448 });
449
450 + // Grows the first page until hidden entries can no longer crowd visible ones out of it.
451 + const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* <I>(
452 + query: string,
453 + pageSize: number,
454 + operation: "directorySearch" | "fileSearch" | "mixedSearch",
455 + execute: (pageSize: number) => Result<SearchPage<I>>,
456 + pathOf: (item: I) => string,
457 + ): Effect.fn.Return<SearchPage<I>, WorkspaceSearchIndexSearchFailed> {
458 + let size = pageSize;
459 + while (true) {
460 + const fetchSize = size;
461 + const result = yield* runSearch(query, fetchSize, operation, () => execute(fetchSize));
462 + if (ignoredPaths.size === 0) return result;
463 + const items = result.items.filter((item) => !isIgnoredPath(ignoredPaths, pathOf(item)));
464 + if (
465 + items.length >= pageSize ||
466 + result.items.length >= result.totalMatched ||
467 + fetchSize >= WORKSPACE_INDEX_PAGE_SIZE
468 + ) {
469 + return {
470 + items,
471 + totalMatched: result.totalMatched - (result.items.length - items.length),
472 + };
473 + }
474 + size = Math.min(WORKSPACE_INDEX_PAGE_SIZE, fetchSize * 4);
475 + }
476 + });
477 +
478 const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
479 "WorkspaceSearchIndex.refresh",
480 )(function* () {
@@ -427,13 +506,18 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
506 cause,
507 }),
508 ).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void }));
509 + ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd);
510 });
511
512 const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
513 function* () {
514 const incompleteBeforeQuery = hasIncompleteInitialScan();
435 - const result = yield* runSearch("", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", () =>
436 - finder.mixedSearch("", { pageSize: WORKSPACE_INDEX_PAGE_SIZE }),
515 + const result = yield* searchVisible(
516 + "",
517 + WORKSPACE_INDEX_PAGE_SIZE,
518 + "mixedSearch",
519 + (pageSize) => finder.mixedSearch("", { pageSize }),
520 + mixedItemPath,
521 );
522 const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES);
523 const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) =>
@@ -456,19 +540,31 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
540 ) {
541 const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1);
542 if (kind === "file" || imageOnly) {
459 - const result = yield* runSearch(query, pageSize, "fileSearch", () =>
460 - finder.fileSearch(query, { pageSize }),
543 + const result = yield* searchVisible(
544 + query,
545 + pageSize,
546 + "fileSearch",
547 + (size) => finder.fileSearch(query, { pageSize: size }),
548 + (item) => item.relativePath,
549 );
550 return mapFileSearchResult(result, limit, imageOnly);
551 }
552 if (kind === "directory") {
465 - const result = yield* runSearch(query, pageSize, "directorySearch", () =>
466 - finder.directorySearch(query, { pageSize }),
553 + const result = yield* searchVisible(
554 + query,
555 + pageSize,
556 + "directorySearch",
557 + (size) => finder.directorySearch(query, { pageSize: size }),
558 + (item) => item.relativePath,
559 );
560 return mapDirectorySearchResult(result, limit);
561 }
470 - const result = yield* runSearch(query, pageSize, "mixedSearch", () =>
471 - finder.mixedSearch(query, { pageSize }),
562 + const result = yield* searchVisible(
563 + query,
564 + pageSize,
565 + "mixedSearch",
566 + (size) => finder.mixedSearch(query, { pageSize: size }),
567 + mixedItemPath,
568 );
569 return mapMixedSearchResult(result, limit);
570 });
@@ -523,6 +619,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
619 }
620
621 for (const match of result.items) {
622 + if (isIgnoredPath(ignoredPaths, match.relativePath)) continue;
623 const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter(
624 (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range),
625 );