The best way to control Claude Code, Codex and more from any computer or phone
fix(server): include symlinks in workspace search (#24578)
Julius Marminge committed
Nov 13, 2026 at 10:04 UTC
76d0fff5fd030eb654ef8316e84c5bf3c3616578
5 files changed
+989
-6
apps/server/src/workspace/IgnorePatterns.test.ts
new
+56
@@ -0,0 +1,56 @@
1
+import { describe, expect, it } from "@effect/vitest";
2
+
3
+import { parseIgnoreFile } from "./IgnorePatterns.ts";
4
+
5
+const decide = (contents: string, path: string, isFolder = false) =>
6
+ parseIgnoreFile(contents).decide(path, isFolder);
7
+
8
+describe("parseIgnoreFile", () => {
9
+ it("lets the last matching pattern decide, including negations", () => {
10
+ expect(decide("*keep*\n!*keep*\n", "sub/keep-link.ts")).toBe("included");
11
+ expect(decide("!*keep*\n*keep*\n", "sub/keep-link.ts")).toBe("ignored");
12
+ expect(decide("# comment\n\nother\n", "keep.ts")).toBeUndefined();
13
+ });
14
+
15
+ it("anchors patterns that contain a slash and matches bare names at any depth", () => {
16
+ expect(decide("link", "a/b/link")).toBe("ignored");
17
+ expect(decide("/link", "a/link")).toBeUndefined();
18
+ expect(decide("/link", "link")).toBe("ignored");
19
+ expect(decide("a/link", "x/a/link")).toBeUndefined();
20
+ expect(decide("a/*.ts", "a/b/c.ts")).toBeUndefined();
21
+ expect(decide("link", "blink")).toBeUndefined();
22
+ expect(decide("link", "a/blink")).toBeUndefined();
23
+ });
24
+
25
+ it("supports globstars, classes, escapes, and folder-only patterns", () => {
26
+ expect(decide("**/gen/*.ts", "gen/a.ts")).toBe("ignored");
27
+ expect(decide("a/**/b", "a/x/y/b")).toBe("ignored");
28
+ expect(decide("a/**/b", "a/b")).toBe("ignored");
29
+ expect(decide("foo/**/bar", "foo/xbar")).toBeUndefined();
30
+ expect(decide("foo/**/bar", "foo/x/ybar")).toBeUndefined();
31
+ expect(decide("out/**", "out/x/y")).toBe("ignored");
32
+ expect(decide("link-[0-9]", "link-7")).toBe("ignored");
33
+ expect(decide("link-[!0-9]", "link-7")).toBeUndefined();
34
+ expect(decide("link-?", "link-/")).toBeUndefined();
35
+ expect(decide("\\!bang\n\\#hash\nstar\\*", "!bang")).toBe("ignored");
36
+ expect(decide("\\!bang\n\\#hash\nstar\\*", "#hash")).toBe("ignored");
37
+ expect(decide("\\!bang\n\\#hash\nstar\\*", "starx")).toBeUndefined();
38
+ expect(decide("dist/", "dist")).toBeUndefined();
39
+ expect(decide("dist/", "dist", true)).toBe("ignored");
40
+ });
41
+
42
+ it("matches adversarial patterns in linear time", () => {
43
+ // A backtracking regex takes seconds to reject this pair; the vitest timeout catches that.
44
+ const pattern = `${"a*".repeat(20)}b\n`;
45
+ expect(decide(pattern, `${"a".repeat(200)}c`)).toBeUndefined();
46
+ expect(decide(pattern.repeat(50), `${"a".repeat(1_000)}c`)).toBeUndefined();
47
+ expect(decide(pattern, `${"a".repeat(200)}b`)).toBe("ignored");
48
+ });
49
+
50
+ it("parses unclosed classes in linear time", () => {
51
+ // Rescanning the rest of the line for each `[` stalls for over a minute here; linear takes ms.
52
+ expect(decide("[".repeat(30_000), "[".repeat(30_000))).toBe("ignored");
53
+ expect(decide(`${"[".repeat(30_000)}]`, "x")).toBeUndefined();
54
+ expect(decide("[a][", "a[")).toBe("ignored");
55
+ });
56
+});
apps/server/src/workspace/IgnorePatterns.ts
new
+170
@@ -0,0 +1,170 @@
1
+/**
2
+ * Gitignore-syntax matching for `.ignore` files, which git cannot evaluate for us. Patterns come
3
+ * from the repository, so each runs as an NFA over the path (O(pattern × path)): a backtracking
4
+ * regex stalls for seconds on `a*a*a*…b`. POSIX `[:class:]` names are not supported.
5
+ */
6
+
7
+type Token =
8
+ | { readonly type: "char"; readonly char: string }
9
+ | { readonly type: "one" }
10
+ | { readonly type: "class"; readonly negated: boolean; readonly ranges: ReadonlyArray<string> }
11
+ | { readonly type: "star" }
12
+ | { readonly type: "any" }
13
+ /** A leading or inner `**` segment: nothing, or anything ending in `/`. */
14
+ | { readonly type: "folders" };
15
+
16
+interface Pattern {
17
+ readonly tokens: ReadonlyArray<Token>;
18
+ readonly negated: boolean;
19
+ readonly folderOnly: boolean;
20
+}
21
+
22
+export type IgnoreDecision = "ignored" | "included";
23
+
24
+function parseClass(chars: ReadonlyArray<string>, start: number) {
25
+ let index = start + 1;
26
+ const negated = chars[index] === "!" || chars[index] === "^";
27
+ if (negated) index += 1;
28
+ const ranges: string[] = [];
29
+ for (let first = true; index < chars.length; first = false) {
30
+ let char = chars[index]!;
31
+ if (char === "]" && !first)
32
+ return { token: { type: "class", negated, ranges } as const, end: index };
33
+ if (char === "\\" && index + 1 < chars.length) char = chars[++index]!;
34
+ if (chars[index + 1] === "-" && index + 2 < chars.length && chars[index + 2] !== "]") {
35
+ let last = chars[index + 2]!;
36
+ index += 2;
37
+ if (last === "\\" && index + 1 < chars.length) last = chars[++index]!;
38
+ ranges.push(char, last);
39
+ } else {
40
+ ranges.push(char, char);
41
+ }
42
+ index += 1;
43
+ }
44
+ return undefined;
45
+}
46
+
47
+function parseLine(rawLine: string): Pattern | undefined {
48
+ let line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
49
+ // Trailing spaces are dropped unless escaped.
50
+ while (line.endsWith(" ") && !line.endsWith("\\ ")) line = line.slice(0, -1);
51
+ if (line === "" || line.startsWith("#")) return undefined;
52
+ const negated = line.startsWith("!");
53
+ if (negated) line = line.slice(1);
54
+ else if (line.startsWith("\\!") || line.startsWith("\\#")) line = line.slice(1);
55
+ const folderOnly = line.endsWith("/") && !line.endsWith("\\/");
56
+ if (folderOnly) line = line.slice(0, -1);
57
+ if (line === "") return undefined;
58
+ const anchored = line.slice(0, -1).includes("/");
59
+ if (line.startsWith("/")) line = line.slice(1);
60
+
61
+ const chars = Array.from(line);
62
+ const tokens: Token[] = anchored ? [] : [{ type: "folders" }];
63
+ // A `[` that never closes leaves every later `[` unclosed too, so one failed scan settles them
64
+ // all; rescanning the suffix for each would make `[[[[…` quadratic.
65
+ let classesClose = true;
66
+ for (let index = 0; index < chars.length; index += 1) {
67
+ const char = chars[index]!;
68
+ if (char === "*" && chars[index + 1] === "*") {
69
+ const atSegmentStart = index === 0 || chars[index - 1] === "/";
70
+ const next = chars[index + 2];
71
+ if (atSegmentStart && next === "/") {
72
+ tokens.push({ type: "folders" });
73
+ index += 2;
74
+ continue;
75
+ }
76
+ if (atSegmentStart && next === undefined) {
77
+ tokens.push({ type: "any" });
78
+ index += 1;
79
+ continue;
80
+ }
81
+ }
82
+ if (char === "*") tokens.push({ type: "star" });
83
+ else if (char === "?") tokens.push({ type: "one" });
84
+ else if (char === "[") {
85
+ const parsed = classesClose ? parseClass(chars, index) : undefined;
86
+ if (!parsed) classesClose = false;
87
+ if (parsed) {
88
+ tokens.push(parsed.token);
89
+ index = parsed.end;
90
+ } else tokens.push({ type: "char", char });
91
+ } else if (char === "\\" && index + 1 < chars.length) {
92
+ tokens.push({ type: "char", char: chars[++index]! });
93
+ } else tokens.push({ type: "char", char });
94
+ }
95
+ return { tokens, negated, folderOnly };
96
+}
97
+
98
+/**
99
+ * States are token positions. A `folders` token that has consumed part of a segment is `~position`
100
+ * instead: it may only finish after the next `/`, so an inner globstar never matches `xbar` as `bar`.
101
+ */
102
+function matches(tokens: ReadonlyArray<Token>, path: ReadonlyArray<string>): boolean {
103
+ let states = new Set<number>();
104
+ const add = (set: Set<number>, position: number) => {
105
+ // Wildcards also match nothing, so reaching one reaches the token after it.
106
+ for (; position <= tokens.length && !set.has(position); position += 1) {
107
+ set.add(position);
108
+ const type = tokens[position]?.type;
109
+ if (type !== "star" && type !== "any" && type !== "folders") return;
110
+ }
111
+ };
112
+ add(states, 0);
113
+ for (const char of path) {
114
+ const next = new Set<number>();
115
+ for (const state of states) {
116
+ const position = state < 0 ? ~state : state;
117
+ const token = tokens[position];
118
+ if (!token) continue;
119
+ switch (token.type) {
120
+ case "char":
121
+ if (token.char === char) add(next, position + 1);
122
+ break;
123
+ case "one":
124
+ if (char !== "/") add(next, position + 1);
125
+ break;
126
+ case "class": {
127
+ let inRange = false;
128
+ for (let index = 0; index < token.ranges.length && !inRange; index += 2) {
129
+ inRange = token.ranges[index]! <= char && char <= token.ranges[index + 1]!;
130
+ }
131
+ if (char !== "/" && inRange !== token.negated) add(next, position + 1);
132
+ break;
133
+ }
134
+ case "star":
135
+ if (char !== "/") add(next, position);
136
+ break;
137
+ case "any":
138
+ add(next, position);
139
+ break;
140
+ case "folders":
141
+ if (char === "/") add(next, position);
142
+ else next.add(~position);
143
+ break;
144
+ }
145
+ }
146
+ if (next.size === 0) return false;
147
+ states = next;
148
+ }
149
+ return states.has(tokens.length);
150
+}
151
+
152
+export interface IgnoreFile {
153
+ /** The last pattern that matches decides; `undefined` when none does. */
154
+ readonly decide: (relativePath: string, isFolder: boolean) => IgnoreDecision | undefined;
155
+}
156
+
157
+export function parseIgnoreFile(contents: string): IgnoreFile {
158
+ const patterns = contents.split("\n").flatMap((line) => parseLine(line) ?? []);
159
+ return {
160
+ decide: (relativePath, isFolder) => {
161
+ const path = Array.from(relativePath);
162
+ for (let index = patterns.length - 1; index >= 0; index -= 1) {
163
+ const pattern = patterns[index]!;
164
+ if (pattern.folderOnly && !isFolder) continue;
165
+ if (matches(pattern.tokens, path)) return pattern.negated ? "included" : "ignored";
166
+ }
167
+ return undefined;
168
+ },
169
+ };
170
+}
apps/server/src/workspace/WorkspaceEntries.test.ts
+127
@@ -546,6 +546,133 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
546
}),
547
);
548
549
+ it.effect("indexes symlinked files and directories inside the workspace", () =>
550
+ Effect.gen(function* () {
551
+ if ((yield* HostProcess.Platform) === "win32") return;
552
+ const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-" });
553
+ const fileSystem = yield* FileSystem.FileSystem;
554
+ const path = yield* Path.Path;
555
+ yield* writeTextFile(cwd, "src/index.ts", "export {};");
556
+ yield* fileSystem.symlink("src", path.join(cwd, "src-link"));
557
+ yield* fileSystem.symlink("src/index.ts", path.join(cwd, "index-link.ts"));
558
+ const symlinkEntries = [
559
+ { kind: "directory", path: "src-link" },
560
+ { kind: "file", path: "index-link.ts" },
561
+ ];
562
+
563
+ const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
564
+ const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
565
+ const listed = yield* workspaceEntries.list({ cwd });
566
+
567
+ for (const entry of symlinkEntries) {
568
+ expect(searched.entries).toContainEqual(expect.objectContaining(entry));
569
+ expect(listed.entries).toContainEqual(expect.objectContaining(entry));
570
+ }
571
+ }),
572
+ );
573
+ it.effect("skips symlinks that ignore rules hide or that leave the workspace", () =>
574
+ Effect.gen(function* () {
575
+ if ((yield* HostProcess.Platform) === "win32") return;
576
+ const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true });
577
+ const outside = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-outside-" });
578
+ const fileSystem = yield* FileSystem.FileSystem;
579
+ const path = yield* Path.Path;
580
+ yield* writeTextFile(cwd, "src/index.ts", "export {};");
581
+ yield* writeTextFile(cwd, ".ignore", "custom-link\n");
582
+ yield* fileSystem.symlink("src", path.join(cwd, "custom-link"));
583
+ yield* fileSystem.symlink("src/index.ts", path.join(cwd, "tracked-link.ts"));
584
+ yield* fileSystem.symlink("src", path.join(cwd, "ignored-link"));
585
+ yield* fileSystem.symlink("src", path.join(cwd, "kept-link"));
586
+ yield* fileSystem.symlink(outside, path.join(cwd, "outside-link"));
587
+ yield* git(cwd, ["add", "tracked-link.ts"]);
588
+ yield* writeTextFile(cwd, ".gitignore", "tracked-link.ts\nignored-link\n");
589
+ // A submodule path makes `git check-ignore` refuse the whole batch without --no-index.
590
+ const submodule = path.join(cwd, "vendor/sub");
591
+ yield* writeTextFile(submodule, "a.ts", "export {};");
592
+ yield* git(submodule, ["init"]);
593
+ yield* git(submodule, ["add", "a.ts"]);
594
+ yield* git(submodule, [
595
+ "-c",
596
+ "user.name=t",
597
+ "-c",
598
+ "user.email=t@t",
599
+ "commit",
600
+ "-m",
601
+ "init",
602
+ ]);
603
+ yield* git(cwd, ["add", "vendor/sub"]);
604
+ yield* fileSystem.symlink("a.ts", path.join(submodule, "sublink.ts"));
605
+
606
+ const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
607
+ const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
608
+ const listed = yield* workspaceEntries.list({ cwd });
609
+
610
+ for (const entries of [searched.entries, listed.entries]) {
611
+ const paths = entries.map((entry) => entry.path);
612
+ expect(paths).toContain("kept-link");
613
+ expect(paths).toContain("vendor/sub/sublink.ts");
614
+ expect(paths).not.toContain("custom-link");
615
+ expect(paths).not.toContain("tracked-link.ts");
616
+ expect(paths).not.toContain("ignored-link");
617
+ expect(paths).not.toContain("outside-link");
618
+ }
619
+ }),
620
+ );
621
+ it.effect("lets the nearest .ignore rule decide a symlink, and git's rules still hide it", () =>
622
+ Effect.gen(function* () {
623
+ if ((yield* HostProcess.Platform) === "win32") return;
624
+ const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true });
625
+ const fileSystem = yield* FileSystem.FileSystem;
626
+ const path = yield* Path.Path;
627
+ yield* writeTextFile(cwd, ".gitignore", "*keep*\n");
628
+ yield* writeTextFile(cwd, ".ignore", "!*keep*\n*hide*\n");
629
+ yield* writeTextFile(cwd, "sub/.ignore", "!*hide*\n");
630
+ yield* writeTextFile(cwd, "keep-real.ts", "export {};");
631
+ yield* writeTextFile(cwd, "sub/hide-real.ts", "export {};");
632
+ yield* fileSystem.symlink("keep-real.ts", path.join(cwd, "keep-link.ts"));
633
+ yield* fileSystem.symlink("hide-real.ts", path.join(cwd, "sub/hide-link.ts"));
634
+
635
+ const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
636
+ const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
637
+ const listed = yield* workspaceEntries.list({ cwd });
638
+
639
+ for (const entries of [searched.entries, listed.entries]) {
640
+ const paths = entries.map((entry) => entry.path);
641
+ expect(paths).toContain("sub/hide-link.ts");
642
+ expect(paths).not.toContain("keep-link.ts");
643
+ }
644
+ // The real files are shown or hidden by the same rules.
645
+ const listedPaths = listed.entries.map((entry) => entry.path);
646
+ expect(listedPaths).toContain("sub/hide-real.ts");
647
+ expect(listedPaths).not.toContain("keep-real.ts");
648
+ }),
649
+ );
650
+ it.effect("matches .ignore rules against whole names and treats folder links as links", () =>
651
+ Effect.gen(function* () {
652
+ if ((yield* HostProcess.Platform) === "win32") return;
653
+ const cwd = yield* makeTempDir({ prefix: "t3code-workspace-symlinks-", git: true });
654
+ const fileSystem = yield* FileSystem.FileSystem;
655
+ const path = yield* Path.Path;
656
+ yield* writeTextFile(cwd, "src/index.ts", "export {};");
657
+ yield* writeTextFile(cwd, ".gitignore", "blink\n");
658
+ // `!link` must not re-include `blink`, and `folder-link/` does not apply to a link.
659
+ yield* writeTextFile(cwd, ".ignore", "!link\nfolder-link/\n");
660
+ yield* fileSystem.symlink("src", path.join(cwd, "blink"));
661
+ yield* fileSystem.symlink("src", path.join(cwd, "folder-link"));
662
+ yield* writeTextFile(cwd, "sub/blink", "regular");
663
+
664
+ const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
665
+ const searched = yield* searchWorkspaceEntries({ cwd, query: "link", limit: 100 });
666
+ const listed = yield* workspaceEntries.list({ cwd });
667
+
668
+ for (const entries of [searched.entries, listed.entries]) {
669
+ const paths = entries.map((entry) => entry.path);
670
+ expect(paths).not.toContain("blink");
671
+ expect(paths).not.toContain("sub/blink");
672
+ expect(paths).toContain("folder-link");
673
+ }
674
+ }),
675
+ );
676
it.effect("finds a path inside a gitignored nested git worktree only by its exact path", () =>
677
Effect.gen(function* () {
678
const cwd = yield* makeTempDir({ prefix: "t3code-workspace-worktree-", git: true });
apps/server/src/workspace/WorkspaceSearchIndex.test.ts
+248
@@ -4,14 +4,18 @@ import {
4
type GrepCursor,
5
type GrepOptions,
6
type GrepResult,
7
+ type WatchEvent,
8
} from "@ff-labs/fff-node";
9
import * as NodeServices from "@effect/platform-node/NodeServices";
10
import { afterEach, expect, it } from "@effect/vitest";
11
import * as HostProcess from "@t3tools/shared/HostProcess";
12
+import * as Deferred from "effect/Deferred";
13
import * as Effect from "effect/Effect";
14
+import * as Fiber from "effect/Fiber";
15
import * as FileSystem from "effect/FileSystem";
16
import * as Layer from "effect/Layer";
17
import * as Path from "effect/Path";
18
+import * as TestClock from "effect/testing/TestClock";
19
import { vi } from "vite-plus/test";
20
21
import * as VcsProcess from "../vcs/VcsProcess.ts";
@@ -55,6 +59,8 @@ it.effect("filters image searches before applying the result limit", () =>
59
const finder = {
60
destroy: vi.fn(),
61
waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
62
+ isScanning: () => false,
63
+ watch: () => ({ ok: true as const, value: () => undefined }),
64
fileSearch,
65
} as unknown as FileFinder;
66
vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -101,6 +107,8 @@ it.effect("fetches past a first page of Git-ignored results", () =>
107
const finder = {
108
destroy: vi.fn(),
109
waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
110
+ isScanning: () => false,
111
+ watch: () => ({ ok: true as const, value: () => undefined }),
112
fileSearch,
113
} as unknown as FileFinder;
114
vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -123,6 +131,8 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
131
const finder = {
132
destroy: vi.fn(),
133
waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
134
+ isScanning: () => false,
135
+ watch: () => ({ ok: true as const, value: () => undefined }),
136
fileSearch: vi.fn(() => ({
137
ok: true as const,
138
value: { items, scores: [], totalMatched: 1, totalFiles: 1 },
@@ -185,6 +195,8 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
195
const finder = {
196
destroy: vi.fn(),
197
waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
198
+ isScanning: () => false,
199
+ watch: () => ({ ok: true as const, value: () => undefined }),
200
grep,
201
} as unknown as FileFinder;
202
vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
@@ -215,6 +227,242 @@ it.effect("continues whole-word searches after a filtered grep page", () =>
227
).pipe(Effect.provide(testLayer)),
228
);
229
230
+function fakeSymlinkFinder(
231
+ directories: ReadonlyArray<string>,
232
+ options: {
233
+ readonly indexed?: ReadonlyArray<{
234
+ readonly path: string;
235
+ readonly kind: "file" | "directory";
236
+ }>;
237
+ readonly watchFailures?: number;
238
+ } = {},
239
+) {
240
+ const watchers: Array<(events: ReadonlyArray<WatchEvent>) => void> = [];
241
+ let watchFailures = options.watchFailures ?? 0;
242
+ // fff's pageIndex is an offset into the results, not a page number.
243
+ const directorySearch = vi.fn(
244
+ (_query: string, searchOptions?: { pageIndex?: number; pageSize?: number }) => {
245
+ const offset = searchOptions?.pageIndex ?? 0;
246
+ const items = directories.slice(offset, offset + (searchOptions?.pageSize ?? 100));
247
+ return {
248
+ ok: true as const,
249
+ value: {
250
+ items: items.map((relativePath) => ({ relativePath })),
251
+ totalMatched: directories.length,
252
+ },
253
+ };
254
+ },
255
+ );
256
+ const indexedItems = (options.indexed ?? []).map((entry) => ({
257
+ type: entry.kind,
258
+ item: { relativePath: entry.path, fileName: entry.path.split("/").at(-1) },
259
+ }));
260
+ const finder = {
261
+ destroy: vi.fn(),
262
+ waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })),
263
+ isScanning: () => false,
264
+ watch: (callback: (events: ReadonlyArray<WatchEvent>) => void) => {
265
+ if (watchFailures > 0) {
266
+ watchFailures -= 1;
267
+ throw new Error("File system watcher is not ready");
268
+ }
269
+ watchers.push(callback);
270
+ return { ok: true as const, value: () => undefined };
271
+ },
272
+ directorySearch,
273
+ fileSearch: () => {
274
+ const items = indexedItems.flatMap((entry) => (entry.type === "file" ? [entry.item] : []));
275
+ return { ok: true as const, value: { items, scores: [], totalMatched: items.length } };
276
+ },
277
+ mixedSearch: () => ({
278
+ ok: true as const,
279
+ value: { items: indexedItems, totalMatched: indexedItems.length },
280
+ }),
281
+ } as unknown as FileFinder;
282
+ vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder });
283
+ return {
284
+ directorySearch,
285
+ notifyWatchers: (events: ReadonlyArray<WatchEvent>) =>
286
+ watchers.forEach((watcher) => watcher(events)),
287
+ };
288
+}
289
+
290
+const listedPaths = (searchIndex: WorkspaceSearchIndex.WorkspaceSearchIndex["Service"]) =>
291
+ searchIndex.list().pipe(Effect.map((result) => result.entries.map((entry) => entry.path)));
292
+
293
+it.effect("reads symlinks again after the watcher reports a change", () =>
294
+ Effect.scoped(
295
+ Effect.gen(function* () {
296
+ if ((yield* HostProcess.Platform) === "win32") return;
297
+ const fileSystem = yield* FileSystem.FileSystem;
298
+ const path = yield* Path.Path;
299
+ const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" });
300
+ const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" });
301
+ yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), "");
302
+ yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts"));
303
+ const { directorySearch, notifyWatchers } = fakeSymlinkFinder([]);
304
+
305
+ const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
306
+ const concurrent = yield* Effect.all(
307
+ Array.from({ length: 8 }, () => listedPaths(searchIndex)),
308
+ { concurrency: "unbounded" },
309
+ );
310
+ yield* fileSystem.remove(path.join(cwd, "old-link.ts"));
311
+ yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts"));
312
+ const beforeNotice = yield* listedPaths(searchIndex);
313
+ // Editing a file cannot move a link, so it does not cost a rescan.
314
+ notifyWatchers([{ kind: "modified", path: path.join(cwd, "a.ts") }]);
315
+ const afterEdit = yield* listedPaths(searchIndex);
316
+ notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]);
317
+ const afterNotice = yield* listedPaths(searchIndex);
318
+ // fff reports a link replaced in place as modified.
319
+ yield* fileSystem.remove(path.join(cwd, "new-link.ts"));
320
+ yield* fileSystem.symlink(outside, path.join(cwd, "new-link.ts"));
321
+ notifyWatchers([{ kind: "modified", path: path.join(cwd, "new-link.ts") }]);
322
+ const afterReplace = yield* listedPaths(searchIndex);
323
+
324
+ expect(concurrent).toEqual(Array.from({ length: 8 }, () => ["old-link.ts"]));
325
+ expect(beforeNotice).toEqual(["old-link.ts"]);
326
+ expect(afterEdit).toEqual(["old-link.ts"]);
327
+ expect(afterNotice).toEqual(["new-link.ts"]);
328
+ expect(afterReplace).toEqual([]);
329
+ expect(directorySearch).toHaveBeenCalledTimes(3);
330
+ }),
331
+ ).pipe(Effect.provide(testLayer)),
332
+);
333
+
334
+it.effect("answers from the last symlink scan while a slow rescan runs", () =>
335
+ Effect.scoped(
336
+ Effect.gen(function* () {
337
+ if ((yield* HostProcess.Platform) === "win32") return;
338
+ const fileSystem = yield* FileSystem.FileSystem;
339
+ const path = yield* Path.Path;
340
+ const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-stale-" });
341
+ yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), "");
342
+ yield* fileSystem.symlink("a.ts", path.join(cwd, "old-link.ts"));
343
+ const { notifyWatchers } = fakeSymlinkFinder([]);
344
+ // Holding git's ignore check open keeps the rescan from finishing.
345
+ const gate = yield* Deferred.make<void>();
346
+ let holdIgnoreChecks = false;
347
+ const vcsProcess = yield* VcsProcess.VcsProcess;
348
+ const searchIndex = yield* WorkspaceSearchIndex.make(cwd).pipe(
349
+ Effect.provideService(VcsProcess.VcsProcess, {
350
+ ...vcsProcess,
351
+ run: (input) =>
352
+ holdIgnoreChecks && input.operation === "WorkspaceSearchIndex.listGitIgnored"
353
+ ? Deferred.await(gate).pipe(Effect.andThen(vcsProcess.run(input)))
354
+ : vcsProcess.run(input),
355
+ }),
356
+ );
357
+ expect(yield* listedPaths(searchIndex)).toEqual(["old-link.ts"]);
358
+
359
+ holdIgnoreChecks = true;
360
+ yield* fileSystem.symlink("a.ts", path.join(cwd, "new-link.ts"));
361
+ notifyWatchers([{ kind: "created", path: path.join(cwd, "new-link.ts") }]);
362
+ const duringRescan = yield* Effect.forkChild(listedPaths(searchIndex));
363
+ yield* TestClock.adjust("1 second");
364
+ expect(yield* Fiber.join(duringRescan)).toEqual(["old-link.ts"]);
365
+
366
+ yield* Deferred.succeed(gate, undefined);
367
+ expect(yield* listedPaths(searchIndex)).toEqual(["new-link.ts", "old-link.ts"]);
368
+ }),
369
+ ).pipe(Effect.provide(testLayer)),
370
+);
371
+
372
+it.effect("keeps symlinks once the watcher subscribes after starting late", () =>
373
+ Effect.scoped(
374
+ Effect.gen(function* () {
375
+ if ((yield* HostProcess.Platform) === "win32") return;
376
+ const fileSystem = yield* FileSystem.FileSystem;
377
+ const path = yield* Path.Path;
378
+ const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-watch-" });
379
+ yield* fileSystem.writeFileString(path.join(cwd, "a.ts"), "");
380
+ yield* fileSystem.symlink("a.ts", path.join(cwd, "link.ts"));
381
+ const { directorySearch } = fakeSymlinkFinder([], { watchFailures: 2 });
382
+
383
+ const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
384
+ const scansAfterRound: number[] = [];
385
+ for (let round = 0; round < 5; round += 1) {
386
+ expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]);
387
+ expect(yield* listedPaths(searchIndex)).toEqual(["link.ts"]);
388
+ scansAfterRound.push(directorySearch.mock.calls.length);
389
+ yield* TestClock.adjust("1 minute");
390
+ }
391
+
392
+ // An unwatched scan serves reads only briefly; the third has a watcher and is kept.
393
+ expect(scansAfterRound).toEqual([1, 2, 3, 3, 3]);
394
+ }),
395
+ ).pipe(Effect.provide(testLayer)),
396
+);
397
+
398
+it.effect(
399
+ "lists a symlink the watcher indexed once, with the link's kind, and drops rejected ones",
400
+ () =>
401
+ Effect.scoped(
402
+ Effect.gen(function* () {
403
+ if ((yield* HostProcess.Platform) === "win32") return;
404
+ const fileSystem = yield* FileSystem.FileSystem;
405
+ const path = yield* Path.Path;
406
+ const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-dupe-" });
407
+ const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-outside-" });
408
+ yield* fileSystem.makeDirectory(path.join(cwd, "src"));
409
+ yield* fileSystem.symlink("src", path.join(cwd, "src-link"));
410
+ // A link the scan rejects stays out even when the watcher indexed it.
411
+ yield* fileSystem.symlink(outside, path.join(cwd, "outside-link"));
412
+ fakeSymlinkFinder([], {
413
+ indexed: [
414
+ { path: "outside-link", kind: "file" },
415
+ { path: "src-link", kind: "file" },
416
+ { path: "link-file.ts", kind: "file" },
417
+ ],
418
+ });
419
+
420
+ const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
421
+ const searched = yield* searchIndex.search("link", 10);
422
+ const files = yield* searchIndex.search("link", 1, "file");
423
+ const listed = yield* searchIndex.list();
424
+
425
+ expect(searched.entries).toEqual([
426
+ { path: "src-link", kind: "directory" },
427
+ { path: "link-file.ts", kind: "file" },
428
+ ]);
429
+ // The dropped link does not take the only slot.
430
+ expect(files).toEqual({
431
+ entries: [{ path: "link-file.ts", kind: "file" }],
432
+ truncated: false,
433
+ });
434
+ expect(listed.entries).toEqual([
435
+ { path: "link-file.ts", kind: "file" },
436
+ { path: "src-link", kind: "directory" },
437
+ ]);
438
+ }),
439
+ ).pipe(Effect.provide(testLayer)),
440
+);
441
+
442
+it.effect("finds symlinks in folders past the first page of the index", () =>
443
+ Effect.scoped(
444
+ Effect.gen(function* () {
445
+ if ((yield* HostProcess.Platform) === "win32") return;
446
+ const fileSystem = yield* FileSystem.FileSystem;
447
+ const path = yield* Path.Path;
448
+ const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-symlink-pages-" });
449
+ yield* fileSystem.makeDirectory(path.join(cwd, "late"));
450
+ yield* fileSystem.writeFileString(path.join(cwd, "late/a.ts"), "");
451
+ yield* fileSystem.symlink("a.ts", path.join(cwd, "late/link.ts"));
452
+ // Folders fff lists but that are gone by the time they are read are skipped.
453
+ const firstPage = Array.from({ length: 1_000 }, (_, index) => `missing-${index}`);
454
+ const { directorySearch } = fakeSymlinkFinder([...firstPage, "late"]);
455
+
456
+ const searchIndex = yield* WorkspaceSearchIndex.make(cwd);
457
+
458
+ expect(yield* listedPaths(searchIndex)).toEqual(["late", "late/link.ts"]);
459
+ expect(directorySearch.mock.calls.map(([, options]) => options?.pageIndex)).toEqual([
460
+ 0, 1_000,
461
+ ]);
462
+ }),
463
+ ).pipe(Effect.provide(testLayer)),
464
+);
465
+
466
// fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there.
467
it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")(
468
"refuses to index a workspace rooted at the home directory",
apps/server/src/workspace/WorkspaceSearchIndex.ts
+388
-6
@@ -1,3 +1,5 @@
1
+// @effect-diagnostics nodeBuiltinImport:off - readdir with file types avoids an lstat per child
2
+import * as NodeFSP from "node:fs/promises";
3
import * as NodeModule from "node:module";
4
5
import type {
@@ -7,11 +9,18 @@ import type {
9
GrepCursor,
10
MixedItem,
11
Result,
12
+ WatchEvent,
13
+ WatchUnsubscribe,
14
} from "@ff-labs/fff-node";
15
+import * as Clock from "effect/Clock";
16
import * as Context from "effect/Context";
17
+import * as Deferred from "effect/Deferred";
18
import * as Effect from "effect/Effect";
19
+import * as FileSystem from "effect/FileSystem";
20
import * as Layer from "effect/Layer";
21
import * as LayerMap from "effect/LayerMap";
22
+import * as Option from "effect/Option";
23
+import * as Path from "effect/Path";
24
import * as Schema from "effect/Schema";
25
26
import type {
@@ -24,8 +33,10 @@ import type {
33
} from "@t3tools/contracts";
34
import * as HostProcess from "@t3tools/shared/HostProcess";
35
import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
36
+import { scoreSubsequenceMatch } from "@t3tools/shared/searchRanking";
37
38
import * as VcsProcess from "../vcs/VcsProcess.ts";
39
+import { type IgnoreDecision, type IgnoreFile, parseIgnoreFile } from "./IgnorePatterns.ts";
40
41
// fff-node's native library stays external; single-executables require disk packages instead of importing them.
42
const requireForFff = NodeModule.createRequire(import.meta.url);
@@ -38,6 +49,13 @@ const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
49
const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
50
const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
51
const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
52
+const SYMLINK_SCAN_CONCURRENCY = 16;
53
+// A scan no watcher can invalidate is reused this long, so typing does not rescan every folder.
54
+const UNWATCHED_SYMLINK_SCAN_TTL_MS = 2_000;
55
+// A rescan that outlasts this leaves the read on the previous scan.
56
+const SYMLINK_RESCAN_WAIT = "5 millis";
57
+// Each page is a synchronous native call; small pages let searches run between them.
58
+const SYMLINK_DIRECTORY_PAGE_SIZE = 1_000;
59
60
export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError<WorkspaceSearchIndexCreateFailed>()(
61
"WorkspaceSearchIndexCreateFailed",
@@ -290,6 +308,135 @@ function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEn
308
return [...entryByPath.values()];
309
}
310
311
+interface ScannedDirectory {
312
+ readonly directory: string;
313
+ readonly links: ReadonlyArray<ProjectEntry>;
314
+ /** Every symlink in the folder, including ones dropped for leaving the workspace. */
315
+ readonly linkPaths: ReadonlyArray<string>;
316
+ /** A nested repository (or the workspace's own), whose ignore rules own what is below it. */
317
+ readonly isRepository: boolean;
318
+ readonly ignoreFile: string | undefined;
319
+}
320
+
321
+const joinEntryPath = (directory: string, name: string) =>
322
+ directory ? `${directory}/${name}` : name;
323
+
324
+/**
325
+ * fff skips symlinks, and `followSymlinks` drops file links, dedupes folder links, and walks out
326
+ * of the workspace. So links are read from the folders fff already indexed (ignore rules applied),
327
+ * without descending into them, and kept only when they resolve inside the workspace.
328
+ */
329
+const scanDirectory = Effect.fn("WorkspaceSearchIndex.scanDirectory")(function* (
330
+ fileSystem: FileSystem.FileSystem,
331
+ path: Path.Path,
332
+ cwd: string,
333
+ root: string,
334
+ directory: string,
335
+) {
336
+ // Effect's readDirectory returns bare names; one lstat per child would cost far more.
337
+ const children = yield* Effect.promise(() =>
338
+ NodeFSP.readdir(path.join(cwd, directory), { withFileTypes: true }).catch(() => []),
339
+ );
340
+ const links: ProjectEntry[] = [];
341
+ const linkPaths: string[] = [];
342
+ let isRepository = false;
343
+ let ignoreFile: string | undefined;
344
+ for (const child of children) {
345
+ if (child.name === ".git") isRepository = true;
346
+ if (child.name === ".ignore" && child.isFile()) {
347
+ ignoreFile = yield* fileSystem
348
+ .readFileString(path.join(cwd, directory, child.name))
349
+ .pipe(Effect.orElseSucceed(() => undefined));
350
+ }
351
+ if (!child.isSymbolicLink()) continue;
352
+ const entryPath = joinEntryPath(directory, child.name);
353
+ linkPaths.push(entryPath);
354
+ const absolutePath = path.join(cwd, entryPath);
355
+ const target = yield* Effect.all([
356
+ fileSystem.realPath(absolutePath),
357
+ fileSystem.stat(absolutePath),
358
+ ]).pipe(Effect.orElseSucceed(() => undefined));
359
+ if (!target) continue;
360
+ const relative = path.relative(root, target[0]);
361
+ if (
362
+ relative === ".." ||
363
+ relative.startsWith(`..${path.sep}`) ||
364
+ path.isAbsolute(relative) ||
365
+ relative.split(path.sep).includes(".git") ||
366
+ (target[1].type !== "Directory" && target[1].type !== "File")
367
+ ) {
368
+ continue;
369
+ }
370
+ links.push({ path: entryPath, kind: target[1].type === "Directory" ? "directory" : "file" });
371
+ }
372
+ return { directory, links, linkPaths, isRepository, ignoreFile } satisfies ScannedDirectory;
373
+});
374
+
375
+/** Whether `entryPath` is `directory` itself or below it. */
376
+const isWithin = (entryPath: string, directory: string) =>
377
+ directory === "" || entryPath.startsWith(`${directory}/`);
378
+
379
+/**
380
+ * fff reads `.ignore` files alongside git's rules, and like ripgrep the nearest one that matches
381
+ * decides, ahead of any `.gitignore`. `ignoreFiles` is ordered deepest folder first.
382
+ */
383
+function decideByIgnoreFiles(
384
+ entry: ProjectEntry,
385
+ ignoreFiles: ReadonlyArray<{ readonly directory: string; readonly file: IgnoreFile }>,
386
+): IgnoreDecision | undefined {
387
+ for (const { directory, file } of ignoreFiles) {
388
+ if (!isWithin(entry.path, directory) || entry.path === directory) continue;
389
+ const relative = directory ? entry.path.slice(directory.length + 1) : entry.path;
390
+ // Like git and ripgrep, a link is never a folder to a folder-only rule, whatever it targets.
391
+ const decision = file.decide(relative, false);
392
+ if (decision) return decision;
393
+ }
394
+ return undefined;
395
+}
396
+
397
+/** Places each symlink before the first index result that matches the query less closely. */
398
+function mergeSymlinkMatches(
399
+ indexed: ProjectSearchEntriesResult,
400
+ linkPaths: ReadonlySet<string>,
401
+ symlinks: ReadonlyArray<ProjectEntry>,
402
+ query: string,
403
+ limit: number,
404
+): ProjectSearchEntriesResult {
405
+ if (linkPaths.size === 0) return indexed;
406
+ // fff's watcher indexes links created after its scan, under a kind of its own choosing, so a
407
+ // path it already returned takes the link's kind (or drops out of a kind-filtered search).
408
+ const symlinkByPath = new Map(symlinks.map((entry) => [entry.path, entry]));
409
+ const score = (entry: ProjectEntry) =>
410
+ scoreSubsequenceMatch(entry.path.toLowerCase(), query) ?? Number.POSITIVE_INFINITY;
411
+ const indexedPaths = new Set(indexed.entries.map((entry) => entry.path));
412
+ const pending = symlinks
413
+ .flatMap((entry) => {
414
+ const entryScore = score(entry);
415
+ return Number.isFinite(entryScore) && !indexedPaths.has(entry.path)
416
+ ? [{ entry, score: entryScore }]
417
+ : [];
418
+ })
419
+ .toSorted((left, right) => left.score - right.score);
420
+ const entries: ProjectEntry[] = [];
421
+ let nextPending = 0;
422
+ for (const indexedEntry of indexed.entries) {
423
+ const entry = linkPaths.has(indexedEntry.path)
424
+ ? symlinkByPath.get(indexedEntry.path)
425
+ : indexedEntry;
426
+ if (!entry) continue;
427
+ const entryScore = score(entry);
428
+ while (nextPending < pending.length && pending[nextPending]!.score < entryScore) {
429
+ entries.push(pending[nextPending++]!.entry);
430
+ }
431
+ entries.push(entry);
432
+ }
433
+ for (; nextPending < pending.length; nextPending += 1) entries.push(pending[nextPending]!.entry);
434
+ return {
435
+ entries: entries.slice(0, limit),
436
+ truncated: indexed.truncated || entries.length > limit,
437
+ };
438
+}
439
+
440
// fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file.
441
const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* (
442
vcsProcess: VcsProcess.VcsProcess["Service"],
@@ -401,6 +548,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
548
variant: WorkspaceSearchIndexVariant = "paths",
549
) {
550
const hostIsWindows = yield* HostProcess.isWindows;
551
+ const fileSystem = yield* FileSystem.FileSystem;
552
+ const path = yield* Path.Path;
553
const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) =>
554
Effect.try({
555
try: () => finder.destroy(),
@@ -460,6 +609,225 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
609
return result.value;
610
});
611
612
+ const root = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => cwd));
613
+
614
+ const listIndexedDirectories = Effect.fn("WorkspaceSearchIndex.listIndexedDirectories")(
615
+ function* () {
616
+ const directories: string[] = [];
617
+ // fff's pageIndex is an offset into the results, not a page number.
618
+ for (let pageIndex = 0; ; pageIndex += SYMLINK_DIRECTORY_PAGE_SIZE) {
619
+ const result = yield* Effect.sync(() => {
620
+ try {
621
+ return finder.directorySearch("", { pageIndex, pageSize: SYMLINK_DIRECTORY_PAGE_SIZE });
622
+ } catch {
623
+ return undefined;
624
+ }
625
+ });
626
+ if (!result?.ok) return directories;
627
+ for (const item of result.value.items) {
628
+ const directory = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
629
+ if (directory) directories.push(directory);
630
+ }
631
+ if (result.value.items.length < SYMLINK_DIRECTORY_PAGE_SIZE) return directories;
632
+ yield* Effect.yieldNow;
633
+ }
634
+ },
635
+ );
636
+
637
+ /** Ignore rules belong to the repository that owns a path, so each one answers for its own. */
638
+ const listGitIgnored = Effect.fn("WorkspaceSearchIndex.listGitIgnored")(function* (
639
+ repository: string,
640
+ entries: ReadonlyArray<ProjectEntry>,
641
+ ) {
642
+ const relativePaths = entries.map((entry) =>
643
+ repository ? entry.path.slice(repository.length + 1) : entry.path,
644
+ );
645
+ const repositoryCwd = path.join(cwd, repository);
646
+ const result = yield* vcsProcess
647
+ .run({
648
+ operation: "WorkspaceSearchIndex.listGitIgnored",
649
+ command: "git",
650
+ // fff applies ignore rules to tracked paths too, so the index is not consulted.
651
+ args: ["-c", "core.fsmonitor=false", "check-ignore", "--no-index", "-z", "--stdin"],
652
+ cwd: repositoryCwd,
653
+ stdin: `${relativePaths.join("\0")}\0`,
654
+ allowNonZeroExit: true,
655
+ timeoutMs: 10_000,
656
+ maxOutputBytes: 16 * 1024 * 1024,
657
+ })
658
+ .pipe(Effect.orElseSucceed(() => undefined));
659
+ // Exit 1 means nothing matched.
660
+ if (result?.exitCode === 1) return [];
661
+ if (result?.exitCode !== 0 || result.stdoutTruncated) {
662
+ // Outside git no rules apply (git exits before reading stdin, so the cause is lost).
663
+ // Any other failure hides the repository's links rather than leaking ones it ignores.
664
+ const insideRepository = yield* vcsProcess
665
+ .run({
666
+ operation: "WorkspaceSearchIndex.listGitIgnored",
667
+ command: "git",
668
+ args: ["rev-parse", "--git-dir"],
669
+ cwd: repositoryCwd,
670
+ timeoutMs: 10_000,
671
+ })
672
+ .pipe(
673
+ Effect.as(true),
674
+ Effect.orElseSucceed(() => false),
675
+ );
676
+ return insideRepository ? entries.map((entry) => entry.path) : [];
677
+ }
678
+ return result.stdout
679
+ .split("\0")
680
+ .filter(Boolean)
681
+ .map((relativePath) => (repository ? `${repository}/${relativePath}` : relativePath));
682
+ });
683
+
684
+ const scanSymlinkEntries = Effect.fn("WorkspaceSearchIndex.scanSymlinkEntries")(function* () {
685
+ const indexComplete = !finder.isScanning();
686
+ const directories = yield* listIndexedDirectories();
687
+ const scanned = yield* Effect.forEach(
688
+ ["", ...directories],
689
+ (directory) => scanDirectory(fileSystem, path, cwd, root, directory),
690
+ { concurrency: SYMLINK_SCAN_CONCURRENCY },
691
+ );
692
+ const candidates = scanned.flatMap((directory) => directory.links);
693
+ const linkPaths = new Set(scanned.flatMap((directory) => directory.linkPaths));
694
+ if (candidates.length === 0) return { entries: candidates, linkPaths, indexComplete };
695
+
696
+ const ignoreFiles = scanned
697
+ .flatMap(({ directory, ignoreFile }) =>
698
+ ignoreFile === undefined ? [] : [{ directory, file: parseIgnoreFile(ignoreFile) }],
699
+ )
700
+ .toSorted((left, right) => right.directory.length - left.directory.length);
701
+ const decisions = new Map(
702
+ candidates.map((entry) => [entry.path, decideByIgnoreFiles(entry, ignoreFiles)]),
703
+ );
704
+ const repositories = [
705
+ "",
706
+ ...scanned.flatMap(({ directory, isRepository }) =>
707
+ isRepository && directory ? [directory] : [],
708
+ ),
709
+ ].toSorted((left, right) => right.length - left.length);
710
+ const entriesByRepository = Map.groupBy(
711
+ candidates.filter((entry) => decisions.get(entry.path) === undefined),
712
+ (entry) =>
713
+ repositories.find(
714
+ (repository) => isWithin(entry.path, repository) && entry.path !== repository,
715
+ ) ?? "",
716
+ );
717
+ const ignored = new Set(
718
+ (yield* Effect.forEach(
719
+ entriesByRepository,
720
+ ([repository, entries]) => listGitIgnored(repository, entries),
721
+ { concurrency: 4 },
722
+ )).flat(),
723
+ );
724
+ return {
725
+ entries: candidates.filter((entry) =>
726
+ decisions.get(entry.path) === undefined
727
+ ? !ignored.has(entry.path)
728
+ : decisions.get(entry.path) === "included",
729
+ ),
730
+ linkPaths,
731
+ indexComplete,
732
+ };
733
+ });
734
+
735
+ // One scan runs at a time and outlives the reads that started it. Only the first read waits for
736
+ // it in full: after that a change or expiry marks the result stale, and reads wait briefly for
737
+ // the rescan before answering from the last finished one, so file churn cannot stall search.
738
+ type SymlinkScan = Effect.Success<ReturnType<typeof scanSymlinkEntries>>;
739
+ let lastSymlinkScan: SymlinkScan | undefined;
740
+ let lastSymlinkScanExpiresAt: number | undefined;
741
+ let pendingSymlinkScan: Deferred.Deferred<SymlinkScan> | undefined;
742
+ let symlinkScanStale = false;
743
+ let knownLinkPaths: ReadonlySet<string> = new Set();
744
+ const invalidateSymlinkScan = () => {
745
+ symlinkScanStale = true;
746
+ };
747
+ const isKnownLink = (absolutePath: string) =>
748
+ [cwd, root].some((base) =>
749
+ knownLinkPaths.has(toPosixPath(path.relative(base, absolutePath), hostIsWindows)),
750
+ );
751
+ const onWatchBatch = (events: ReadonlyArray<WatchEvent>) => {
752
+ // fff reports a link replaced in place as `modified`, so edits to known links count too.
753
+ const editsOnly = events.every(
754
+ (event) =>
755
+ event.kind === "modified" &&
756
+ !/(^|[\\/])\.(git)?ignore$/.test(event.path) &&
757
+ !isKnownLink(event.path),
758
+ );
759
+ if (!editsOnly) invalidateSymlinkScan();
760
+ };
761
+ // fff refuses subscriptions until its watcher is up, which can trail the initial scan, so
762
+ // each read retries until one is in place.
763
+ let unwatch: WatchUnsubscribe | undefined;
764
+ yield* Effect.addFinalizer(() => Effect.sync(() => unwatch?.()));
765
+ const ensureWatching = () => {
766
+ if (unwatch) return true;
767
+ try {
768
+ const result = finder.watch(onWatchBatch);
769
+ if (result.ok) unwatch = result.value;
770
+ } catch {
771
+ // Not ready yet; the next read tries again.
772
+ }
773
+ return unwatch !== undefined;
774
+ };
775
+ const indexScope = yield* Effect.scope;
776
+ const startSymlinkScan = Effect.fn("WorkspaceSearchIndex.startSymlinkScan")(function* () {
777
+ symlinkScanStale = false;
778
+ // Subscribing before the scan starts means no change during it goes unseen.
779
+ const watched = ensureWatching();
780
+ const done = yield* Deferred.make<SymlinkScan>();
781
+ pendingSymlinkScan = done;
782
+ yield* scanSymlinkEntries().pipe(
783
+ Effect.tap((result) =>
784
+ Effect.gen(function* () {
785
+ lastSymlinkScan = result;
786
+ knownLinkPaths = result.linkPaths;
787
+ // Without a watcher, or before fff has seen every folder, the result is kept only briefly.
788
+ lastSymlinkScanExpiresAt =
789
+ watched && result.indexComplete
790
+ ? undefined
791
+ : (yield* Clock.currentTimeMillis) + UNWATCHED_SYMLINK_SCAN_TTL_MS;
792
+ }),
793
+ ),
794
+ Effect.onExit((exit) =>
795
+ Effect.sync(() => {
796
+ pendingSymlinkScan = undefined;
797
+ }).pipe(Effect.andThen(Deferred.done(done, exit))),
798
+ ),
799
+ Effect.forkIn(indexScope),
800
+ );
801
+ return done;
802
+ });
803
+ const listSymlinkEntries = Effect.fn("WorkspaceSearchIndex.listSymlinkEntries")(function* () {
804
+ if (
805
+ lastSymlinkScanExpiresAt !== undefined &&
806
+ (yield* Clock.currentTimeMillis) >= lastSymlinkScanExpiresAt
807
+ ) {
808
+ symlinkScanStale = true;
809
+ }
810
+ const pending =
811
+ pendingSymlinkScan ??
812
+ (lastSymlinkScan === undefined || symlinkScanStale ? yield* startSymlinkScan() : undefined);
813
+ let result = lastSymlinkScan;
814
+ if (pending && result === undefined) {
815
+ result = yield* Deferred.await(pending);
816
+ } else if (pending) {
817
+ const fresh = yield* Deferred.await(pending).pipe(Effect.timeoutOption(SYMLINK_RESCAN_WAIT));
818
+ if (Option.isSome(fresh)) result = fresh.value;
819
+ }
820
+ if (result === undefined)
821
+ return yield* Effect.die("The symlink scan finished without a result.");
822
+ return {
823
+ // Git's ignore rules hide a link the same way searchVisible hides the files it indexes.
824
+ entries: result.entries.filter(
825
+ (entry) => !isIgnoredPath(ignoredPaths, entry.path, hostIsWindows),
826
+ ),
827
+ linkPaths: result.linkPaths,
828
+ };
829
+ });
830
+
831
// Grows the first page until hidden entries can no longer crowd visible ones out of it.
832
const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* <I>(
833
query: string,
@@ -522,6 +890,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
890
}),
891
).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void }));
892
ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows);
893
+ invalidateSymlinkScan();
894
});
895
896
const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
@@ -535,8 +904,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
904
mixedItemPath,
905
);
906
const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES, hostIsWindows);
538
- const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) =>
539
- left.path.localeCompare(right.path),
907
+ const symlinks = yield* listSymlinkEntries();
908
+ // fff's watcher can index a link the scan rejected; only accepted links are listed.
909
+ const indexed = mapped.entries.filter((entry) => !symlinks.linkPaths.has(entry.path));
910
+ const sortedEntries = withDirectoryAncestors([...indexed, ...symlinks.entries]).toSorted(
911
+ (left, right) => left.path.localeCompare(right.path),
912
);
913
const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES);
914
return {
@@ -553,7 +925,17 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
925
kind: ProjectEntryKind | undefined,
926
imageOnly: boolean | undefined,
927
) {
556
- const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1);
928
+ const { entries: allSymlinks, linkPaths } = yield* listSymlinkEntries();
929
+ const symlinks = allSymlinks.filter((entry) =>
930
+ imageOnly
931
+ ? entry.kind === "file" && isProjectFaviconPath(entry.path)
932
+ : kind === undefined || entry.kind === kind,
933
+ );
934
+ // Index results that are filtered-out links get dropped in the merge, so fetch past them.
935
+ const indexLimit = limit + linkPaths.size - symlinks.length;
936
+ const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, indexLimit + 1);
937
+ const merge = (indexed: ProjectSearchEntriesResult) =>
938
+ mergeSymlinkMatches(indexed, linkPaths, symlinks, query, limit);
939
if (kind === "file" || imageOnly) {
940
const result = yield* searchVisible(
941
query,
@@ -562,7 +944,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
944
(size) => finder.fileSearch(query, { pageSize: size }),
945
(item) => item.relativePath,
946
);
565
- return mapFileSearchResult(result, limit, hostIsWindows, imageOnly);
947
+ return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, imageOnly));
948
}
949
if (kind === "directory") {
950
const result = yield* searchVisible(
@@ -572,7 +954,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
954
(size) => finder.directorySearch(query, { pageSize: size }),
955
(item) => item.relativePath,
956
);
575
- return mapDirectorySearchResult(result, limit, hostIsWindows);
957
+ return merge(mapDirectorySearchResult(result, indexLimit, hostIsWindows));
958
}
959
const result = yield* searchVisible(
960
query,
@@ -581,7 +963,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
963
(size) => finder.mixedSearch(query, { pageSize: size }),
964
mixedItemPath,
965
);
584
- return mapMixedSearchResult(result, limit, hostIsWindows);
966
+ return merge(mapMixedSearchResult(result, indexLimit, hostIsWindows));
967
});
968
969
const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn(