@devries-cameron20 / t3code-1 / commits / 76d0fff5fd
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): include symlinks in workspace search (#24578)

Julius Marminge committed Nov 13, 2026 at 10:04 UTC 76d0fff5fd030eb654ef8316e84c5bf3c3616578
5 files changed +989 -6
apps/server/src/workspace/IgnorePatterns.test.ts new
+56
@@ -0,0 +1,56 @@
1 +import { describe, expect, it } from "@effect/vitest";
2 +
3 +import { parseIgnoreFile } from "./IgnorePatterns.ts";
4 +
5 +const decide = (contents: string, path: string, isFolder = false) =>
6 + parseIgnoreFile(contents).decide(path, isFolder);
7 +
8 +describe("parseIgnoreFile", () => {
9 + it("lets the last matching pattern decide, including negations", () => {
10 + expect(decide("*keep*\n!*keep*\n", "sub/keep-link.ts")).toBe("included");
11 + expect(decide("!*keep*\n*keep*\n", "sub/keep-link.ts")).toBe("ignored");
12 + expect(decide("# comment\n\nother\n", "keep.ts")).toBeUndefined();
13 + });
14 +
15 + it("anchors patterns that contain a slash and matches bare names at any depth", () => {
16 + expect(decide("link", "a/b/link")).toBe("ignored");
17 + expect(decide("/link", "a/link")).toBeUndefined();
18 + expect(decide("/link", "link")).toBe("ignored");
19 + expect(decide("a/link", "x/a/link")).toBeUndefined();
20 + expect(decide("a/*.ts", "a/b/c.ts")).toBeUndefined();
21 + expect(decide("link", "blink")).toBeUndefined();
22 + expect(decide("link", "a/blink")).toBeUndefined();
23 + });
24 +
25 + it("supports globstars, classes, escapes, and folder-only patterns", () => {
26 + expect(decide("**/gen/*.ts", "gen/a.ts")).toBe("ignored");
27 + expect(decide("a/**/b", "a/x/y/b")).toBe("ignored");
28 + expect(decide("a/**/b", "a/b")).toBe("ignored");
29 + expect(decide("foo/**/bar", "foo/xbar")).toBeUndefined();
30 + expect(decide("foo/**/bar", "foo/x/ybar")).toBeUndefined();
31 + expect(decide("out/**", "out/x/y")).toBe("ignored");
32 + expect(decide("link-[0-9]", "link-7")).toBe("ignored");
33 + expect(decide("link-[!0-9]", "link-7")).toBeUndefined();
34 + expect(decide("link-?", "link-/")).toBeUndefined();
35 + expect(decide("\\!bang\n\\#hash\nstar\\*", "!bang")).toBe("ignored");
36 + expect(decide("\\!bang\n\\#hash\nstar\\*", "#hash")).toBe("ignored");
37 + expect(decide("\\!bang\n\\#hash\nstar\\*", "starx")).toBeUndefined();
38 + expect(decide("dist/", "dist")).toBeUndefined();
39 + expect(decide("dist/", "dist", true)).toBe("ignored");
40 + });
41 +
42 + it("matches adversarial patterns in linear time", () => {
43 + // A backtracking regex takes seconds to reject this pair; the vitest timeout catches that.
44 + const pattern = `${"a*".repeat(20)}b\n`;
45 + expect(decide(pattern, `${"a".repeat(200)}c`)).toBeUndefined();
46 + expect(decide(pattern.repeat(50), `${"a".repeat(1_000)}c`)).toBeUndefined();
47 + expect(decide(pattern, `${"a".repeat(200)}b`)).toBe("ignored");
48 + });
49 +
50 + it("parses unclosed classes in linear time", () => {
51 + // Rescanning the rest of the line for each `[` stalls for over a minute here; linear takes ms.
52 + expect(decide("[".repeat(30_000), "[".repeat(30_000))).toBe("ignored");
53 + expect(decide(`${"[".repeat(30_000)}]`, "x")).toBeUndefined();
54 + expect(decide("[a][", "a[")).toBe("ignored");
55 + });
56 +});
apps/server/src/workspace/IgnorePatterns.ts new
+170
@@ -0,0 +1,170 @@
1 +/**
2 + * Gitignore-syntax matching for `.ignore` files, which git cannot evaluate for us. Patterns come
3 + * from the repository, so each runs as an NFA over the path (O(pattern × path)): a backtracking
4 + * regex stalls for seconds on `a*a*a*…b`. POSIX `[:class:]` names are not supported.
5 + */
6 +
7 +type Token =
8 + | { readonly type: "char"; readonly char: string }
9 + | { readonly type: "one" }
10 + | { readonly type: "class"; readonly negated: boolean; readonly ranges: ReadonlyArray<string> }
11 + | { readonly type: "star" }
12 + | { readonly type: "any" }
13 + /** A leading or inner `**` segment: nothing, or anything ending in `/`. */
14 + | { readonly type: "folders" };
15 +
16 +interface Pattern {
17 + readonly tokens: ReadonlyArray<Token>;
18 + readonly negated: boolean;
19 + readonly folderOnly: boolean;
20 +}
21 +
22 +export type IgnoreDecision = "ignored" | "included";
23 +
24 +function parseClass(chars: ReadonlyArray<string>, start: number) {
25 + let index = start + 1;
26 + const negated = chars[index] === "!" || chars[index] === "^";
27 + if (negated) index += 1;
28 + const ranges: string[] = [];
29 + for (let first = true; index < chars.length; first = false) {
30 + let char = chars[index]!;
31 + if (char === "]" && !first)
32 + return { token: { type: "class", negated, ranges } as const, end: index };
33 + if (char === "\\" && index + 1 < chars.length) char = chars[++index]!;
34 + if (chars[index + 1] === "-" && index + 2 < chars.length && chars[index + 2] !== "]") {
35 + let last = chars[index + 2]!;
36 + index += 2;
37 + if (last === "\\" && index + 1 < chars.length) last = chars[++index]!;
38 + ranges.push(char, last);
39 + } else {
40 + ranges.push(char, char);
41 + }
42 + index += 1;
43 + }
44 + return undefined;
45 +}
46 +
47 +function parseLine(rawLine: string): Pattern | undefined {
48 + let line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
49 + // Trailing spaces are dropped unless escaped.
50 + while (line.endsWith(" ") && !line.endsWith("\\ ")) line = line.slice(0, -1);
51 + if (line === "" || line.startsWith("#")) return undefined;
52 + const negated = line.startsWith("!");
53 + if (negated) line = line.slice(1);
54 + else if (line.startsWith("\\!") || line.startsWith("\\#")) line = line.slice(1);
55 + const folderOnly = line.endsWith("/") && !line.endsWith("\\/");
56 + if (folderOnly) line = line.slice(0, -1);
57 + if (line === "") return undefined;
58 + const anchored = line.slice(0, -1).includes("/");
59 + if (line.startsWith("/")) line = line.slice(1);
60 +
61 + const chars = Array.from(line);
62 + const tokens: Token[] = anchored ? [] : [{ type: "folders" }];
63 + // A `[` that never closes leaves every later `[` unclosed too, so one failed scan settles them
64 + // all; rescanning the suffix for each would make `[[[[…` quadratic.
65 + let classesClose = true;
66 + for (let index = 0; index < chars.length; index += 1) {
67 + const char = chars[index]!;
68 + if (char === "*" && chars[index + 1] === "*") {
69 + const atSegmentStart = index === 0 || chars[index - 1] === "/";
70 + const next = chars[index + 2];
71 + if (atSegmentStart && next === "/") {
72 + tokens.push({ type: "folders" });
73 + index += 2;
74 + continue;
75 + }
76 + if (atSegmentStart && next === undefined) {
77 + tokens.push({ type: "any" });
78 + index += 1;
79 + continue;
80 + }
81 + }
82 + if (char === "*") tokens.push({ type: "star" });
83 + else if (char === "?") tokens.push({ type: "one" });
84 + else if (char === "[") {
85 + const parsed = classesClose ? parseClass(chars, index) : undefined;
86 + if (!parsed) classesClose = false;
87 + if (parsed) {
88 + tokens.push(parsed.token);
89 + index = parsed.end;
90 + } else tokens.push({ type: "char", char });
91 + } else if (char === "\\" && index + 1 < chars.length) {
92 + tokens.push({ type: "char", char: chars[++index]! });
93 + } else tokens.push({ type: "char", char });
94 + }
95 + return { tokens, negated, folderOnly };
96 +}
97 +
98 +/**
99 + * States are token positions. A `folders` token that has consumed part of a segment is `~position`
100 + * instead: it may only finish after the next `/`, so an inner globstar never matches `xbar` as `bar`.
101 + */
102 +function matches(tokens: ReadonlyArray<Token>, path: ReadonlyArray<string>): boolean {
103 + let states = new Set<number>();
104 + const add = (set: Set<number>, position: number) => {
105 + // Wildcards also match nothing, so reaching one reaches the token after it.
106 + for (; position <= tokens.length && !set.has(position); position += 1) {
107 + set.add(position);
108 + const type = tokens[position]?.type;
109 + if (type !== "star" && type !== "any" && type !== "folders") return;
110 + }
111 + };
112 + add(states, 0);
113 + for (const char of path) {
114 + const next = new Set<number>();
115 + for (const state of states) {
116 + const position = state < 0 ? ~state : state;
117 + const token = tokens[position];
118 + if (!token) continue;
119 + switch (token.type) {
120 + case "char":
121 + if (token.char === char) add(next, position + 1);
122 + break;
123 + case "one":
124 + if (char !== "/") add(next, position + 1);
125 + break;
126 + case "class": {
127 + let inRange = false;
128 + for (let index = 0; index < token.ranges.length && !inRange; index += 2) {
129 + inRange = token.ranges[index]! <= char && char <= token.ranges[index + 1]!;
130 + }
131 + if (char !== "/" && inRange !== token.negated) add(next, position + 1);
132 + break;
133 + }
134 + case "star":
135 + if (char !== "/") add(next, position);
136 + break;
137 + case "any":
138 + add(next, position);
139 + break;
140 + case "folders":
141 + if (char === "/") add(next, position);
142 + else next.add(~position);
143 + break;
144 + }
145 + }
146 + if (next.size === 0) return false;
147 + states = next;
148 + }
149 + return states.has(tokens.length);
150 +}
151 +
152 +export interface IgnoreFile {
153 + /** The last pattern that matches decides; `undefined` when none does. */
154 + readonly decide: (relativePath: string, isFolder: boolean) => IgnoreDecision | undefined;
155 +}
156 +
157 +export function parseIgnoreFile(contents: string): IgnoreFile {
158 + const patterns = contents.split("\n").flatMap((line) => parseLine(line) ?? []);
159 + return {
160 + decide: (relativePath, isFolder) => {
161 + const path = Array.from(relativePath);
162 + for (let index = patterns.length - 1; index >= 0; index -= 1) {
163 + const pattern = patterns[index]!;
164 + if (pattern.folderOnly && !isFolder) continue;
165 + if (matches(pattern.tokens, path)) return pattern.negated ? "included" : "ignored";
166 + }
167 + return undefined;
168 + },
169 + };
170 +}
apps/server/src/workspace/WorkspaceEntries.test.ts
+127
@@ -546,6 +546,133 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
546 }),
547 );
548
549 + it.effect("indexes symlinked files and directories inside the workspace", () =>
550 + Effect.gen(function* () {
551 + if ((yield* HostProcess.Platform) === "win32") return;
552 + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-" });
553 + const fileSystem = yield* FileSystem.FileSystem;
554 + const path = yield* Path.Path;
555 + yield* writeTextFile(cwd, "src/index.ts", "export {};");
556 + yield* fileSystem.symlink("src", path.join(cwd, "src-link"));
557 + yield* fileSystem.symlink("src/index.ts", path.join(cwd, "index-link.ts"));
558 + const symlinkEntries = [
559 + { kind: "directory", path: "src-link" },
560 + { kind: "file", path: "index-link.ts" },
561 + ];
562 +
563 + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
564 + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
565 + const listed = yield* workspaceEntries.list({ cwd });
566 +
567 + for (const entry of symlinkEntries) {
568 + expect(searched.entries).toContainEqual(expect.objectContaining(entry));
569 + expect(listed.entries).toContainEqual(expect.objectContaining(entry));
570 + }
571 + }),
572 + );
573 + it.effect("skips symlinks that ignore rules hide or that leave the workspace", () =>
574 + Effect.gen(function* () {
575 + if ((yield* HostProcess.Platform) === "win32") return;
576 + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true });
577 + const outside = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-outside-" });
578 + const fileSystem = yield* FileSystem.FileSystem;
579 + const path = yield* Path.Path;
580 + yield* writeTextFile(cwd, "src/index.ts", "export {};");
581 + yield* writeTextFile(cwd, ".ignore", "custom-link\n");
582 + yield* fileSystem.symlink("src", path.join(cwd, "custom-link"));
583 + yield* fileSystem.symlink("src/index.ts", path.join(cwd, "tracked-link.ts"));
584 + yield* fileSystem.symlink("src", path.join(cwd, "ignored-link"));
585 + yield* fileSystem.symlink("src", path.join(cwd, "kept-link"));
586 + yield* fileSystem.symlink(outside, path.join(cwd, "outside-link"));
587 + yield* git(cwd, ["add", "tracked-link.ts"]);
588 + yield* writeTextFile(cwd, ".gitignore", "tracked-link.ts\nignored-link\n");
589 + // A submodule path makes `git check-ignore` refuse the whole batch without --no-index.
590 + const submodule = path.join(cwd, "vendor/sub");
591 + yield* writeTextFile(submodule, "a.ts", "export {};");
592 + yield* git(submodule, ["init"]);
593 + yield* git(submodule, ["add", "a.ts"]);
594 + yield* git(submodule, [
595 + "-c",
596 + "user.name=t",
597 + "-c",
598 + "user.email=t@t",
599 + "commit",
600 + "-m",
601 + "init",
602 + ]);
603 + yield* git(cwd, ["add", "vendor/sub"]);
604 + yield* fileSystem.symlink("a.ts", path.join(submodule, "sublink.ts"));
605 +
606 + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
607 + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
608 + const listed = yield* workspaceEntries.list({ cwd });
609 +
610 + for (const entries of [searched.entries, listed.entries]) {
611 + const paths = entries.map((entry) => entry.path);
612 + expect(paths).toContain("kept-link");
613 + expect(paths).toContain("vendor/sub/sublink.ts");
614 + expect(paths).not.toContain("custom-link");
615 + expect(paths).not.toContain("tracked-link.ts");
616 + expect(paths).not.toContain("ignored-link");
617 + expect(paths).not.toContain("outside-link");
618 + }
619 + }),
620 + );
621 + it.effect("lets the nearest .ignore rule decide a symlink, and git's rules still hide it", () =>
622 + Effect.gen(function* () {
623 + if ((yield* HostProcess.Platform) === "win32") return;
624 + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true });
625 + const fileSystem = yield* FileSystem.FileSystem;
626 + const path = yield* Path.Path;
627 + yield* writeTextFile(cwd, ".gitignore", "*keep*\n");
628 + yield* writeTextFile(cwd, ".ignore", "!*keep*\n*hide*\n");
629 + yield* writeTextFile(cwd, "sub/.ignore", "!*hide*\n");
630 + yield* writeTextFile(cwd, "keep-real.ts", "export {};");
631 + yield* writeTextFile(cwd, "sub/hide-real.ts", "export {};");
632 + yield* fileSystem.symlink("keep-real.ts", path.join(cwd, "keep-link.ts"));
633 + yield* fileSystem.symlink("hide-real.ts", path.join(cwd, "sub/hide-link.ts"));
634 +
635 + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
636 + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
637 + const listed = yield* workspaceEntries.list({ cwd });
638 +
639 + for (const entries of [searched.entries, listed.entries]) {
640 + const paths = entries.map((entry) => entry.path);
641 + expect(paths).toContain("sub/hide-link.ts");
642 + expect(paths).not.toContain("keep-link.ts");
643 + }
644 + // The real files are shown or hidden by the same rules.
645 + const listedPaths = listed.entries.map((entry) => entry.path);
646 + expect(listedPaths).toContain("sub/hide-real.ts");
647 + expect(listedPaths).not.toContain("keep-real.ts");
648 + }),
649 + );
650 + it.effect("matches .ignore rules against whole names and treats folder links as links", () =>
651 + Effect.gen(function* () {
652 + if ((yield* HostProcess.Platform) === "win32") return;
653 + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true });
654 + const fileSystem = yield* FileSystem.FileSystem;
655 + const path = yield* Path.Path;
656 + yield* writeTextFile(cwd, "src/index.ts", "export {};");
657 + yield* writeTextFile(cwd, ".gitignore", "blink\n");
658 + // `!link` must not re-include `blink`, and `folder-link/` does not apply to a link.
659 + yield* writeTextFile(cwd, ".ignore", "!link\nfolder-link/\n");
660 + yield* fileSystem.symlink("src", path.join(cwd, "blink"));
661 + yield* fileSystem.symlink("src", path.join(cwd, "folder-link"));
662 + yield* writeTextFile(cwd, "sub/blink", "regular");
663 +
664 + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
665 + const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
666 + const listed = yield* workspaceEntries.list({ cwd });
667 +
668 + for (const entries of [searched.entries, listed.entries]) {
669 + const paths = entries.map((entry) => entry.path);
670 + expect(paths).not.toContain("blink");
671 + expect(paths).not.toContain("sub/blink");
672 + expect(paths).toContain("folder-link");
673 + }
674 + }),
675 + );
676 it.effect("finds a path inside a gitignored nested git worktree only by its exact path", () =>
677 Effect.gen(function* () {
678 const cwd = yield* makeTempDir({ prefix: "t3code-workspace-worktree-", git: true });
apps/server/src/workspace/WorkspaceSearchIndex.test.ts
+248
@@ -4,14 +4,18 @@ import {
4 type GrepCursor,
5 type GrepOptions,
6 type GrepResult,
7 + type WatchEvent,
8 } from "@ff-labs/fff-node";
9 import * as NodeServices from "@effect/platform-node/NodeServices";
10 import { afterEach, expect, it } from "@effect/vitest";
11 import * as HostProcess from "@t3tools/shared/HostProcess";
12 +import * as Deferred from "effect/Deferred";
13 import * as Effect from "effect/Effect";
14 +import * as Fiber from "effect/Fiber";
15 import * as FileSystem from "effect/FileSystem";
16 import * as Layer from "effect/Layer";
17 import * as Path from "effect/Path";
18 +import * as TestClock from "effect/testing/TestClock";
19 import { vi } from "vite-plus/test";
20
21 import * as VcsProcess from "../vcs/VcsProcess.ts";
@@ -55,6 +59,8 @@ it.effect("filters image searches before applying the result limit", () =>
59 const finder = {
60 destroy: vi.fn(),
61 waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
62 + isScanning: () => false,
63 + watch: () => ({ ok: true as const, value: () => undefined }),
64 fileSearch,
65 } as unknown as FileFinder;
66 vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -101,6 +107,8 @@ it.effect("fetches past a first page of Git-ignored results", () =>
107 const finder = {
108 destroy: vi.fn(),
109 waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
110 + isScanning: () => false,
111 + watch: () => ({ ok: true as const, value: () => undefined }),
112 fileSearch,
113 } as unknown as FileFinder;
114 vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -123,6 +131,8 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
131 const finder = {
132 destroy: vi.fn(),
133 waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
134 + isScanning: () => false,
135 + watch: () => ({ ok: true as const, value: () => undefined }),
136 fileSearch: vi.fn(() => ({
137 ok: true as const,
138 value: { items, scores: [], totalMatched: 1, totalFiles: 1 },
@@ -185,6 +195,8 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
195 const finder = {
196 destroy: vi.fn(),
197 waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
198 + isScanning: () => false,
199 + watch: () => ({ ok: true as const, value: () => undefined }),
200 grep,
201 } as unknown as FileFinder;
202 vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -215,6 +227,242 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
227 ).pipe(Effect.provide(testLayer)),
228 );
229
230 +function fakeSymlinkFinder(
231 + directories: ReadonlyArray<string>,
232 + options: {
233 + readonly indexed?: ReadonlyArray<{
234 + readonly path: string;
235 + readonly kind: "file" | "directory";
236 + }>;
237 + readonly watchFailures?: number;
238 + } = {},
239 +) {
240 + const watchers: Array<(events: ReadonlyArray<WatchEvent>) => void> = [];
241 + let watchFailures = options.watchFailures ?? 0;
242 + // fff's pageIndex is an offset into the results, not a page number.
243 + const directorySearch = vi.fn(
244 + (_query: string, searchOptions?: { pageIndex?: number; pageSize?: number }) => {
245 + const offset = searchOptions?.pageIndex ?? 0;
246 + const items = directories.slice(offset, offset + (searchOptions?.pageSize ?? 100));
247 + return {
248 + ok: true as const,
249 + value: {
250 + items: items.map((relativePath) => ({ relativePath })),
251 + totalMatched: directories.length,
252 + },
253 + };
254 + },
255 + );
256 + const indexedItems = (options.indexed ?? []).map((entry) => ({
257 + type: entry.kind,
258 + item: { relativePath: entry.path, fileName: entry.path.split("/").at(-1) },
259 + }));
260 + const finder = {
261 + destroy: vi.fn(),
262 + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
263 + isScanning: () => false,
264 + watch: (callback: (events: ReadonlyArray<WatchEvent>) => void) => {
265 + if (watchFailures > 0) {
266 + watchFailures -= 1;
267 + throw new Error("File system watcher is not ready");
268 + }
269 + watchers.push(callback);
270 + return { ok: true as const, value: () => undefined };
271 + },
272 + directorySearch,
273 + fileSearch: () => {
274 + const items = indexedItems.flatMap((entry) => (entry.type === "file" ? [entry.item] : []));
275 + return { ok: true as const, value: { items, scores: [], totalMatched: items.length } };
276 + },
277 + mixedSearch: () => ({
278 + ok: true as const,
279 + value: { items: indexedItems, totalMatched: indexedItems.length },
280 + }),
281 + } as unknown as FileFinder;
282 + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
283 + return {
284 + directorySearch,
285 + notifyWatchers: (events: ReadonlyArray<WatchEvent>) =>
286 + watchers.forEach((watcher) => watcher(events)),
287 + };
288 +}
289 +
290 +const listedPaths = (searchIndex: WorkspaceSearchIndex.WorkspaceSearchIndex["Service"]) =>
291 + searchIndex.list().pipe(Effect.map((result) => result.entries.map((entry) => entry.path)));
292 +
293 +it.effect("reads symlinks again after the watcher reports a change", () =>
294 + Effect.scoped(
295 + Effect.gen(function* () {
296 + if ((yield* HostProcess.Platform) === "win32") return;
297 + const fileSystem = yield* FileSystem.FileSystem;
298 + const path = yield* Path.Path;
299 + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" });
300 + const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" });
301 + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), "");
302 + yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts"));
303 + const { directorySearch, notifyWatchers } = fakeSymlinkFinder([]);
304 +
305 + const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
306 + const concurrent = yield* Effect.all(
307 + Array.from({ length: 8 }, () => listedPaths(searchIndex)),
308 + { concurrency: "unbounded" },
309 + );
310 + yield* fileSystem.remove(path.join(cwd, "old-link.ts"));
311 + yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts"));
312 + const beforeNotice = yield* listedPaths(searchIndex);
313 + // Editing a file cannot move a link, so it does not cost a rescan.
314 + notifyWatchers([{ kind: "modified", path: path.join(cwd, "a.ts") }]);
315 + const afterEdit = yield* listedPaths(searchIndex);
316 + notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]);
317 + const afterNotice = yield* listedPaths(searchIndex);
318 + // fff reports a link replaced in place as modified.
319 + yield* fileSystem.remove(path.join(cwd, "new-link.ts"));
320 + yield* fileSystem.symlink(outside, path.join(cwd, "new-link.ts"));
321 + notifyWatchers([{ kind: "modified", path: path.join(cwd, "new-link.ts") }]);
322 + const afterReplace = yield* listedPaths(searchIndex);
323 +
324 + expect(concurrent).toEqual(Array.from({ length: 8 }, () => ["old-link.ts"]));
325 + expect(beforeNotice).toEqual(["old-link.ts"]);
326 + expect(afterEdit).toEqual(["old-link.ts"]);
327 + expect(afterNotice).toEqual(["new-link.ts"]);
328 + expect(afterReplace).toEqual([]);
329 + expect(directorySearch).toHaveBeenCalledTimes(3);
330 + }),
331 + ).pipe(Effect.provide(testLayer)),
332 +);
333 +
334 +it.effect("answers from the last symlink scan while a slow rescan runs", () =>
335 + Effect.scoped(
336 + Effect.gen(function* () {
337 + if ((yield* HostProcess.Platform) === "win32") return;
338 + const fileSystem = yield* FileSystem.FileSystem;
339 + const path = yield* Path.Path;
340 + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-stale-" });
341 + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), "");
342 + yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts"));
343 + const { notifyWatchers } = fakeSymlinkFinder([]);
344 + // Holding git's ignore check open keeps the rescan from finishing.
345 + const gate = yield* Deferred.make<void>();
346 + let holdIgnoreChecks = false;
347 + const vcsProcess = yield* VcsProcess.VcsProcess;
348 + const searchIndex = yield* WorkspaceSearchIndex.make(cwd).pipe(
349 + Effect.provideService(VcsProcess.VcsProcess, {
350 + ...vcsProcess,
351 + run: (input) =>
352 + holdIgnoreChecks && input.operation === "WorkspaceSearchIndex.listGitIgnored"
353 + ? Deferred.await(gate).pipe(Effect.andThen(vcsProcess.run(input)))
354 + : vcsProcess.run(input),
355 + }),
356 + );
357 + expect(yield* listedPaths(searchIndex)).toEqual(["old-link.ts"]);
358 +
359 + holdIgnoreChecks = true;
360 + yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts"));
361 + notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]);
362 + const duringRescan = yield* Effect.forkChild(listedPaths(searchIndex));
363 + yield* TestClock.adjust("1 second");
364 + expect(yield* Fiber.join(duringRescan)).toEqual(["old-link.ts"]);
365 +
366 + yield* Deferred.succeed(gate, undefined);
367 + expect(yield* listedPaths(searchIndex)).toEqual(["new-link.ts", "old-link.ts"]);
368 + }),
369 + ).pipe(Effect.provide(testLayer)),
370 +);
371 +
372 +it.effect("keeps symlinks once the watcher subscribes after starting late", () =>
373 + Effect.scoped(
374 + Effect.gen(function* () {
375 + if ((yield* HostProcess.Platform) === "win32") return;
376 + const fileSystem = yield* FileSystem.FileSystem;
377 + const path = yield* Path.Path;
378 + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" });
379 + yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), "");
380 + yield* fileSystem.symlink("a.ts", path.join(cwd, "link.ts"));
381 + const { directorySearch } = fakeSymlinkFinder([], { watchFailures: 2 });
382 +
383 + const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
384 + const scansAfterRound: number[] = [];
385 + for (let round = 0; round < 5; round += 1) {
386 + expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]);
387 + expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]);
388 + scansAfterRound.push(directorySearch.mock.calls.length);
389 + yield* TestClock.adjust("1 minute");
390 + }
391 +
392 + // An unwatched scan serves reads only briefly; the third has a watcher and is kept.
393 + expect(scansAfterRound).toEqual([1, 2, 3, 3, 3]);
394 + }),
395 + ).pipe(Effect.provide(testLayer)),
396 +);
397 +
398 +it.effect(
399 + "lists a symlink the watcher indexed once, with the link's kind, and drops rejected ones",
400 + () =>
401 + Effect.scoped(
402 + Effect.gen(function* () {
403 + if ((yield* HostProcess.Platform) === "win32") return;
404 + const fileSystem = yield* FileSystem.FileSystem;
405 + const path = yield* Path.Path;
406 + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-dupe-" });
407 + const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" });
408 + yield* fileSystem.makeDirectory(path.join(cwd, "src"));
409 + yield* fileSystem.symlink("src", path.join(cwd, "src-link"));
410 + // A link the scan rejects stays out even when the watcher indexed it.
411 + yield* fileSystem.symlink(outside, path.join(cwd, "outside-link"));
412 + fakeSymlinkFinder([], {
413 + indexed: [
414 + { path: "outside-link", kind: "file" },
415 + { path: "src-link", kind: "file" },
416 + { path: "link-file.ts", kind: "file" },
417 + ],
418 + });
419 +
420 + const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
421 + const searched = yield* searchIndex.search("link", 10);
422 + const files = yield* searchIndex.search("link", 1, "file");
423 + const listed = yield* searchIndex.list();
424 +
425 + expect(searched.entries).toEqual([
426 + { path: "src-link", kind: "directory" },
427 + { path: "link-file.ts", kind: "file" },
428 + ]);
429 + // The dropped link does not take the only slot.
430 + expect(files).toEqual({
431 + entries: [{ path: "link-file.ts", kind: "file" }],
432 + truncated: false,
433 + });
434 + expect(listed.entries).toEqual([
435 + { path: "link-file.ts", kind: "file" },
436 + { path: "src-link", kind: "directory" },
437 + ]);
438 + }),
439 + ).pipe(Effect.provide(testLayer)),
440 +);
441 +
442 +it.effect("finds symlinks in folders past the first page of the index", () =>
443 + Effect.scoped(
444 + Effect.gen(function* () {
445 + if ((yield* HostProcess.Platform) === "win32") return;
446 + const fileSystem = yield* FileSystem.FileSystem;
447 + const path = yield* Path.Path;
448 + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-pages-" });
449 + yield* fileSystem.makeDirectory(path.join(cwd, "late"));
450 + yield* fileSystem.writeFileString(path.join(cwd, "late/a.ts"), "");
451 + yield* fileSystem.symlink("a.ts", path.join(cwd, "late/link.ts"));
452 + // Folders fff lists but that are gone by the time they are read are skipped.
453 + const firstPage = Array.from({ length: 1_000 }, (_, index) => `missing-${index}`);
454 + const { directorySearch } = fakeSymlinkFinder([...firstPage, "late"]);
455 +
456 + const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
457 +
458 + expect(yield* listedPaths(searchIndex)).toEqual(["late", "late/link.ts"]);
459 + expect(directorySearch.mock.calls.map(([, options]) => options?.pageIndex)).toEqual([
460 + 0, 1_000,
461 + ]);
462 + }),
463 + ).pipe(Effect.provide(testLayer)),
464 +);
465 +
466 // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there.
467 it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
468 "refuses to index a workspace rooted at the home directory",
apps/server/src/workspace/WorkspaceSearchIndex.ts
+388 -6
@@ -1,3 +1,5 @@
1 +// @effect-diagnostics nodeBuiltinImport:off - readdir with file types avoids an lstat per child
2 +import * as NodeFSP from "node:fs/promises";
3 import * as NodeModule from "node:module";
4
5 import type {
@@ -7,11 +9,18 @@ import type {
9 GrepCursor,
10 MixedItem,
11 Result,
12 + WatchEvent,
13 + WatchUnsubscribe,
14 } from "@ff-labs/fff-node";
15 +import * as Clock from "effect/Clock";
16 import * as Context from "effect/Context";
17 +import * as Deferred from "effect/Deferred";
18 import * as Effect from "effect/Effect";
19 +import * as FileSystem from "effect/FileSystem";
20 import * as Layer from "effect/Layer";
21 import * as LayerMap from "effect/LayerMap";
22 +import * as Option from "effect/Option";
23 +import * as Path from "effect/Path";
24 import * as Schema from "effect/Schema";
25
26 import type {
@@ -24,8 +33,10 @@ import type {
33 } from "@t3tools/contracts";
34 import * as HostProcess from "@t3tools/shared/HostProcess";
35 import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
36 +import { scoreSubsequenceMatch } from "@t3tools/shared/searchRanking";
37
38 import * as VcsProcess from "../vcs/VcsProcess.ts";
39 +import { type IgnoreDecision, type IgnoreFile, parseIgnoreFile } from "./IgnorePatterns.ts";
40
41 // fff-node's native library stays external; single-executables require disk packages instead of importing them.
42 const requireForFff = NodeModule.createRequire(import.meta.url);
@@ -38,6 +49,13 @@ const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
49 const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
50 const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
51 const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
52 +const SYMLINK_SCAN_CONCURRENCY = 16;
53 +// A scan no watcher can invalidate is reused this long, so typing does not rescan every folder.
54 +const UNWATCHED_SYMLINK_SCAN_TTL_MS = 2_000;
55 +// A rescan that outlasts this leaves the read on the previous scan.
56 +const SYMLINK_RESCAN_WAIT = "5 millis";
57 +// Each page is a synchronous native call; small pages let searches run between them.
58 +const SYMLINK_DIRECTORY_PAGE_SIZE = 1_000;
59
60 export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError<WorkspaceSearchIndexCreateFailed>()(
61 "WorkspaceSearchIndexCreateFailed",
@@ -290,6 +308,135 @@ function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEn
308 return [...entryByPath.values()];
309 }
310
311 +interface ScannedDirectory {
312 + readonly directory: string;
313 + readonly links: ReadonlyArray<ProjectEntry>;
314 + /** Every symlink in the folder, including ones dropped for leaving the workspace. */
315 + readonly linkPaths: ReadonlyArray<string>;
316 + /** A nested repository (or the workspace's own), whose ignore rules own what is below it. */
317 + readonly isRepository: boolean;
318 + readonly ignoreFile: string | undefined;
319 +}
320 +
321 +const joinEntryPath = (directory: string, name: string) =>
322 + directory ? `${directory}/${name}` : name;
323 +
324 +/**
325 + * fff skips symlinks, and `followSymlinks` drops file links, dedupes folder links, and walks out
326 + * of the workspace. So links are read from the folders fff already indexed (ignore rules applied),
327 + * without descending into them, and kept only when they resolve inside the workspace.
328 + */
329 +const scanDirectory = Effect.fn("WorkspaceSearchIndex.scanDirectory")(function* (
330 + fileSystem: FileSystem.FileSystem,
331 + path: Path.Path,
332 + cwd: string,
333 + root: string,
334 + directory: string,
335 +) {
336 + // Effect's readDirectory returns bare names; one lstat per child would cost far more.
337 + const children = yield* Effect.promise(() =>
338 + NodeFSP.readdir(path.join(cwd, directory), { withFileTypes: true }).catch(() => []),
339 + );
340 + const links: ProjectEntry[] = [];
341 + const linkPaths: string[] = [];
342 + let isRepository = false;
343 + let ignoreFile: string | undefined;
344 + for (const child of children) {
345 + if (child.name === ".git") isRepository = true;
346 + if (child.name === ".ignore" && child.isFile()) {
347 + ignoreFile = yield* fileSystem
348 + .readFileString(path.join(cwd, directory, child.name))
349 + .pipe(Effect.orElseSucceed(() => undefined));
350 + }
351 + if (!child.isSymbolicLink()) continue;
352 + const entryPath = joinEntryPath(directory, child.name);
353 + linkPaths.push(entryPath);
354 + const absolutePath = path.join(cwd, entryPath);
355 + const target = yield* Effect.all([
356 + fileSystem.realPath(absolutePath),
357 + fileSystem.stat(absolutePath),
358 + ]).pipe(Effect.orElseSucceed(() => undefined));
359 + if (!target) continue;
360 + const relative = path.relative(root, target[0]);
361 + if (
362 + relative === ".." ||
363 + relative.startsWith(`..${path.sep}`) ||
364 + path.isAbsolute(relative) ||
365 + relative.split(path.sep).includes(".git") ||
366 + (target[1].type !== "Directory" && target[1].type !== "File")
367 + ) {
368 + continue;
369 + }
370 + links.push({ path: entryPath, kind: target[1].type === "Directory" ? "directory" : "file" });
371 + }
372 + return { directory, links, linkPaths, isRepository, ignoreFile } satisfies ScannedDirectory;
373 +});
374 +
375 +/** Whether `entryPath` is `directory` itself or below it. */
376 +const isWithin = (entryPath: string, directory: string) =>
377 + directory === "" || entryPath.startsWith(`${directory}/`);
378 +
379 +/**
380 + * fff reads `.ignore` files alongside git's rules, and like ripgrep the nearest one that matches
381 + * decides, ahead of any `.gitignore`. `ignoreFiles` is ordered deepest folder first.
382 + */
383 +function decideByIgnoreFiles(
384 + entry: ProjectEntry,
385 + ignoreFiles: ReadonlyArray<{ readonly directory: string; readonly file: IgnoreFile }>,
386 +): IgnoreDecision | undefined {
387 + for (const { directory, file } of ignoreFiles) {
388 + if (!isWithin(entry.path, directory) || entry.path === directory) continue;
389 + const relative = directory ? entry.path.slice(directory.length + 1) : entry.path;
390 + // Like git and ripgrep, a link is never a folder to a folder-only rule, whatever it targets.
391 + const decision = file.decide(relative, false);
392 + if (decision) return decision;
393 + }
394 + return undefined;
395 +}
396 +
397 +/** Places each symlink before the first index result that matches the query less closely. */
398 +function mergeSymlinkMatches(
399 + indexed: ProjectSearchEntriesResult,
400 + linkPaths: ReadonlySet<string>,
401 + symlinks: ReadonlyArray<ProjectEntry>,
402 + query: string,
403 + limit: number,
404 +): ProjectSearchEntriesResult {
405 + if (linkPaths.size === 0) return indexed;
406 + // fff's watcher indexes links created after its scan, under a kind of its own choosing, so a
407 + // path it already returned takes the link's kind (or drops out of a kind-filtered search).
408 + const symlinkByPath = new Map(symlinks.map((entry) => [entry.path, entry]));
409 + const score = (entry: ProjectEntry) =>
410 + scoreSubsequenceMatch(entry.path.toLowerCase(), query) ?? Number.POSITIVE_INFINITY;
411 + const indexedPaths = new Set(indexed.entries.map((entry) => entry.path));
412 + const pending = symlinks
413 + .flatMap((entry) => {
414 + const entryScore = score(entry);
415 + return Number.isFinite(entryScore) && !indexedPaths.has(entry.path)
416 + ? [{ entry, score: entryScore }]
417 + : [];
418 + })
419 + .toSorted((left, right) => left.score - right.score);
420 + const entries: ProjectEntry[] = [];
421 + let nextPending = 0;
422 + for (const indexedEntry of indexed.entries) {
423 + const entry = linkPaths.has(indexedEntry.path)
424 + ? symlinkByPath.get(indexedEntry.path)
425 + : indexedEntry;
426 + if (!entry) continue;
427 + const entryScore = score(entry);
428 + while (nextPending < pending.length && pending[nextPending]!.score < entryScore) {
429 + entries.push(pending[nextPending++]!.entry);
430 + }
431 + entries.push(entry);
432 + }
433 + for (; nextPending < pending.length; nextPending += 1) entries.push(pending[nextPending]!.entry);
434 + return {
435 + entries: entries.slice(0, limit),
436 + truncated: indexed.truncated || entries.length > limit,
437 + };
438 +}
439 +
440 // fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file.
441 const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* (
442 vcsProcess: VcsProcess.VcsProcess["Service"],
@@ -401,6 +548,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
548 variant: WorkspaceSearchIndexVariant = "paths",
549 ) {
550 const hostIsWindows = yield* HostProcess.isWindows;
551 + const fileSystem = yield* FileSystem.FileSystem;
552 + const path = yield* Path.Path;
553 const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) =>
554 Effect.try({
555 try: () => finder.destroy(),
@@ -460,6 +609,225 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
609 return result.value;
610 });
611
612 + const root = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => cwd));
613 +
614 + const listIndexedDirectories = Effect.fn("WorkspaceSearchIndex.listIndexedDirectories")(
615 + function* () {
616 + const directories: string[] = [];
617 + // fff's pageIndex is an offset into the results, not a page number.
618 + for (let pageIndex = 0; ; pageIndex += SYMLINK_DIRECTORY_PAGE_SIZE) {
619 + const result = yield* Effect.sync(() => {
620 + try {
621 + return finder.directorySearch("", { pageIndex, pageSize: SYMLINK_DIRECTORY_PAGE_SIZE });
622 + } catch {
623 + return undefined;
624 + }
625 + });
626 + if (!result?.ok) return directories;
627 + for (const item of result.value.items) {
628 + const directory = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
629 + if (directory) directories.push(directory);
630 + }
631 + if (result.value.items.length < SYMLINK_DIRECTORY_PAGE_SIZE) return directories;
632 + yield* Effect.yieldNow;
633 + }
634 + },
635 + );
636 +
637 + /** Ignore rules belong to the repository that owns a path, so each one answers for its own. */
638 + const listGitIgnored = Effect.fn("WorkspaceSearchIndex.listGitIgnored")(function* (
639 + repository: string,
640 + entries: ReadonlyArray<ProjectEntry>,
641 + ) {
642 + const relativePaths = entries.map((entry) =>
643 + repository ? entry.path.slice(repository.length + 1) : entry.path,
644 + );
645 + const repositoryCwd = path.join(cwd, repository);
646 + const result = yield* vcsProcess
647 + .run({
648 + operation: "WorkspaceSearchIndex.listGitIgnored",
649 + command: "git",
650 + // fff applies ignore rules to tracked paths too, so the index is not consulted.
651 + args: ["-c", "core.fsmonitor=false", "check-ignore", "--no-index", "-z", "--stdin"],
652 + cwd: repositoryCwd,
653 + stdin: `${relativePaths.join("\0")}\0`,
654 + allowNonZeroExit: true,
655 + timeoutMs: 10_000,
656 + maxOutputBytes: 16 * 1024 * 1024,
657 + })
658 + .pipe(Effect.orElseSucceed(() => undefined));
659 + // Exit 1 means nothing matched.
660 + if (result?.exitCode === 1) return [];
661 + if (result?.exitCode !== 0 || result.stdoutTruncated) {
662 + // Outside git no rules apply (git exits before reading stdin, so the cause is lost).
663 + // Any other failure hides the repository's links rather than leaking ones it ignores.
664 + const insideRepository = yield* vcsProcess
665 + .run({
666 + operation: "WorkspaceSearchIndex.listGitIgnored",
667 + command: "git",
668 + args: ["rev-parse", "--git-dir"],
669 + cwd: repositoryCwd,
670 + timeoutMs: 10_000,
671 + })
672 + .pipe(
673 + Effect.as(true),
674 + Effect.orElseSucceed(() => false),
675 + );
676 + return insideRepository ? entries.map((entry) => entry.path) : [];
677 + }
678 + return result.stdout
679 + .split("\0")
680 + .filter(Boolean)
681 + .map((relativePath) => (repository ? `${repository}/${relativePath}` : relativePath));
682 + });
683 +
684 + const scanSymlinkEntries = Effect.fn("WorkspaceSearchIndex.scanSymlinkEntries")(function* () {
685 + const indexComplete = !finder.isScanning();
686 + const directories = yield* listIndexedDirectories();
687 + const scanned = yield* Effect.forEach(
688 + ["", ...directories],
689 + (directory) => scanDirectory(fileSystem, path, cwd, root, directory),
690 + { concurrency: SYMLINK_SCAN_CONCURRENCY },
691 + );
692 + const candidates = scanned.flatMap((directory) => directory.links);
693 + const linkPaths = new Set(scanned.flatMap((directory) => directory.linkPaths));
694 + if (candidates.length === 0) return { entries: candidates, linkPaths, indexComplete };
695 +
696 + const ignoreFiles = scanned
697 + .flatMap(({ directory, ignoreFile }) =>
698 + ignoreFile === undefined ? [] : [{ directory, file: parseIgnoreFile(ignoreFile) }],
699 + )
700 + .toSorted((left, right) => right.directory.length - left.directory.length);
701 + const decisions = new Map(
702 + candidates.map((entry) => [entry.path, decideByIgnoreFiles(entry, ignoreFiles)]),
703 + );
704 + const repositories = [
705 + "",
706 + ...scanned.flatMap(({ directory, isRepository }) =>
707 + isRepository && directory ? [directory] : [],
708 + ),
709 + ].toSorted((left, right) => right.length - left.length);
710 + const entriesByRepository = Map.groupBy(
711 + candidates.filter((entry) => decisions.get(entry.path) === undefined),
712 + (entry) =>
713 + repositories.find(
714 + (repository) => isWithin(entry.path, repository) && entry.path !== repository,
715 + ) ?? "",
716 + );
717 + const ignored = new Set(
718 + (yield* Effect.forEach(
719 + entriesByRepository,
720 + ([repository, entries]) => listGitIgnored(repository, entries),
721 + { concurrency: 4 },
722 + )).flat(),
723 + );
724 + return {
725 + entries: candidates.filter((entry) =>
726 + decisions.get(entry.path) === undefined
727 + ? !ignored.has(entry.path)
728 + : decisions.get(entry.path) === "included",
729 + ),
730 + linkPaths,
731 + indexComplete,
732 + };
733 + });
734 +
735 + // One scan runs at a time and outlives the reads that started it. Only the first read waits for
736 + // it in full: after that a change or expiry marks the result stale, and reads wait briefly for
737 + // the rescan before answering from the last finished one, so file churn cannot stall search.
738 + type SymlinkScan = Effect.Success<ReturnType<typeof scanSymlinkEntries>>;
739 + let lastSymlinkScan: SymlinkScan | undefined;
740 + let lastSymlinkScanExpiresAt: number | undefined;
741 + let pendingSymlinkScan: Deferred.Deferred<SymlinkScan> | undefined;
742 + let symlinkScanStale = false;
743 + let knownLinkPaths: ReadonlySet<string> = new Set();
744 + const invalidateSymlinkScan = () => {
745 + symlinkScanStale = true;
746 + };
747 + const isKnownLink = (absolutePath: string) =>
748 + [cwd, root].some((base) =>
749 + knownLinkPaths.has(toPosixPath(path.relative(base, absolutePath), hostIsWindows)),
750 + );
751 + const onWatchBatch = (events: ReadonlyArray<WatchEvent>) => {
752 + // fff reports a link replaced in place as `modified`, so edits to known links count too.
753 + const editsOnly = events.every(
754 + (event) =>
755 + event.kind === "modified" &&
756 + !/(^|[\\/])\.(git)?ignore$/.test(event.path) &&
757 + !isKnownLink(event.path),
758 + );
759 + if (!editsOnly) invalidateSymlinkScan();
760 + };
761 + // fff refuses subscriptions until its watcher is up, which can trail the initial scan, so
762 + // each read retries until one is in place.
763 + let unwatch: WatchUnsubscribe | undefined;
764 + yield* Effect.addFinalizer(() => Effect.sync(() => unwatch?.()));
765 + const ensureWatching = () => {
766 + if (unwatch) return true;
767 + try {
768 + const result = finder.watch(onWatchBatch);
769 + if (result.ok) unwatch = result.value;
770 + } catch {
771 + // Not ready yet; the next read tries again.
772 + }
773 + return unwatch !== undefined;
774 + };
775 + const indexScope = yield* Effect.scope;
776 + const startSymlinkScan = Effect.fn("WorkspaceSearchIndex.startSymlinkScan")(function* () {
777 + symlinkScanStale = false;
778 + // Subscribing before the scan starts means no change during it goes unseen.
779 + const watched = ensureWatching();
780 + const done = yield* Deferred.make<SymlinkScan>();
781 + pendingSymlinkScan = done;
782 + yield* scanSymlinkEntries().pipe(
783 + Effect.tap((result) =>
784 + Effect.gen(function* () {
785 + lastSymlinkScan = result;
786 + knownLinkPaths = result.linkPaths;
787 + // Without a watcher, or before fff has seen every folder, the result is kept only briefly.
788 + lastSymlinkScanExpiresAt =
789 + watched && result.indexComplete
790 + ? undefined
791 + : (yield* Clock.currentTimeMillis) + UNWATCHED_SYMLINK_SCAN_TTL_MS;
792 + }),
793 + ),
794 + Effect.onExit((exit) =>
795 + Effect.sync(() => {
796 + pendingSymlinkScan = undefined;
797 + }).pipe(Effect.andThen(Deferred.done(done, exit))),
798 + ),
799 + Effect.forkIn(indexScope),
800 + );
801 + return done;
802 + });
803 + const listSymlinkEntries = Effect.fn("WorkspaceSearchIndex.listSymlinkEntries")(function* () {
804 + if (
805 + lastSymlinkScanExpiresAt !== undefined &&
806 + (yield* Clock.currentTimeMillis) >= lastSymlinkScanExpiresAt
807 + ) {
808 + symlinkScanStale = true;
809 + }
810 + const pending =
811 + pendingSymlinkScan ??
812 + (lastSymlinkScan === undefined || symlinkScanStale ? yield* startSymlinkScan() : undefined);
813 + let result = lastSymlinkScan;
814 + if (pending && result === undefined) {
815 + result = yield* Deferred.await(pending);
816 + } else if (pending) {
817 + const fresh = yield* Deferred.await(pending).pipe(Effect.timeoutOption(SYMLINK_RESCAN_WAIT));
818 + if (Option.isSome(fresh)) result = fresh.value;
819 + }
820 + if (result === undefined)
821 + return yield* Effect.die("The symlink scan finished without a result.");
822 + return {
823 + // Git's ignore rules hide a link the same way searchVisible hides the files it indexes.
824 + entries: result.entries.filter(
825 + (entry) => !isIgnoredPath(ignoredPaths, entry.path, hostIsWindows),
826 + ),
827 + linkPaths: result.linkPaths,
828 + };
829 + });
830 +
831 // Grows the first page until hidden entries can no longer crowd visible ones out of it.
832 const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* <I>(
833 query: string,
@@ -522,6 +890,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
890 }),
891 ).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void }));
892 ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows);
893 + invalidateSymlinkScan();
894 });
895
896 const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
@@ -535,8 +904,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
904 mixedItemPath,
905 );
906 const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES, hostIsWindows);
538 - const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) =>
539 - left.path.localeCompare(right.path),
907 + const symlinks = yield* listSymlinkEntries();
908 + // fff's watcher can index a link the scan rejected; only accepted links are listed.
909 + const indexed = mapped.entries.filter((entry) => !symlinks.linkPaths.has(entry.path));
910 + const sortedEntries = withDirectoryAncestors([...indexed, ...symlinks.entries]).toSorted(
911 + (left, right) => left.path.localeCompare(right.path),
912 );
913 const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES);
914 return {
@@ -553,7 +925,17 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
925 kind: ProjectEntryKind | undefined,
926 imageOnly: boolean | undefined,
927 ) {
556 - const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1);
928 + const { entries: allSymlinks, linkPaths } = yield* listSymlinkEntries();
929 + const symlinks = allSymlinks.filter((entry) =>
930 + imageOnly
931 + ? entry.kind === "file" && isProjectFaviconPath(entry.path)
932 + : kind === undefined || entry.kind === kind,
933 + );
934 + // Index results that are filtered-out links get dropped in the merge, so fetch past them.
935 + const indexLimit = limit + linkPaths.size - symlinks.length;
936 + const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, indexLimit + 1);
937 + const merge = (indexed: ProjectSearchEntriesResult) =>
938 + mergeSymlinkMatches(indexed, linkPaths, symlinks, query, limit);
939 if (kind === "file" || imageOnly) {
940 const result = yield* searchVisible(
941 query,
@@ -562,7 +944,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
944 (size) => finder.fileSearch(query, { pageSize: size }),
945 (item) => item.relativePath,
946 );
565 - return mapFileSearchResult(result, limit, hostIsWindows, imageOnly);
947 + return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, imageOnly));
948 }
949 if (kind === "directory") {
950 const result = yield* searchVisible(
@@ -572,7 +954,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
954 (size) => finder.directorySearch(query, { pageSize: size }),
955 (item) => item.relativePath,
956 );
575 - return mapDirectorySearchResult(result, limit, hostIsWindows);
957 + return merge(mapDirectorySearchResult(result, indexLimit, hostIsWindows));
958 }
959 const result = yield* searchVisible(
960 query,
@@ -581,7 +963,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
963 (size) => finder.mixedSearch(query, { pageSize: size }),
964 mixedItemPath,
965 );
584 - return mapMixedSearchResult(result, limit, hostIsWindows);
966 + return merge(mapMixedSearchResult(result, indexLimit, hostIsWindows));
967 });
968
969 const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn(