@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 1,440 lines 53.8 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off - readdir with file types avoids an lstat per child
2 import * as NodeFSP from "node:fs/promises";
3 import * as NodeModule from "node:module";
4
5 import type {
6 DirItem,
7 FileItem,
8 FileFinder as FileFinderType,
9 GrepCursor,
10 MixedItem,
11 Result,
12 WatchEvent,
13 WatchUnsubscribe,
14 } from "@ff-labs/fff-node";
15 import * as Clock from "effect/Clock";
16 import * as Context from "effect/Context";
17 import * as Deferred from "effect/Deferred";
18 import * as Effect from "effect/Effect";
19 import * as FileSystem from "effect/FileSystem";
20 import * as Layer from "effect/Layer";
21 import * as LayerMap from "effect/LayerMap";
22 import * as Option from "effect/Option";
23 import * as Path from "effect/Path";
24 import * as Schema from "effect/Schema";
25
26 import type {
27 ProjectEntry,
28 ProjectEntryKind,
29 ProjectListEntriesResult,
30 ProjectSearchContentsInput,
31 ProjectSearchContentsResult,
32 ProjectSearchEntriesResult,
33 } from "@t3tools/contracts";
34 import * as HostProcess from "@t3tools/shared/HostProcess";
35 import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
36 import { scoreSubsequenceMatch } from "@t3tools/shared/searchRanking";
37
38 import * as VcsProcess from "../vcs/VcsProcess.ts";
39 import { type IgnoreDecision, type IgnoreFile, parseIgnoreFile } from "./IgnorePatterns.ts";
40
41 // fff-node's native library stays external; single-executables require disk packages instead of importing them.
42 const requireForFff = NodeModule.createRequire(import.meta.url);
43 const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node");
44
45 // A glob intersects existing extension constraints; fff ORs bare `*.ext` constraints together.
46 const IMAGE_SEARCH_CONSTRAINT = `**/*.{${PROJECT_FAVICON_EXTENSIONS.map((extension) =>
47 extension.slice(1).replace(/[a-z]/g, (letter) => `[${letter}${letter.toUpperCase()}]`),
48 ).join(",")}}`;
49
50 // fff 0.11 disables filename fallback above this match count per base/overflow segment.
51 const FFF_FILENAME_FALLBACK_MAX_MATCHES = 15_000;
52
53 const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
54 const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;
55 const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";
56 const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
57 export const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
58 const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
59 const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
60 const GIT_GREP_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
61 // fff's lineContent cap; matches past it are reported without ranges.
62 const CONTENT_LINE_MAX_LENGTH = 512;
63 const SYMLINK_SCAN_CONCURRENCY = 16;
64 // A scan no watcher can invalidate is reused this long, so typing does not rescan every folder.
65 const UNWATCHED_SYMLINK_SCAN_TTL_MS = 2_000;
66 // A rescan that outlasts this leaves the read on the previous scan.
67 const SYMLINK_RESCAN_WAIT = "5 millis";
68 // Each page is a synchronous native call; small pages let searches run between them.
69 const SYMLINK_DIRECTORY_PAGE_SIZE = 1_000;
70
71 export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError<WorkspaceSearchIndexCreateFailed>()(
72 "WorkspaceSearchIndexCreateFailed",
73 {
74 cwd: Schema.String,
75 reason: Schema.String,
76 cause: Schema.optional(Schema.Defect()),
77 },
78 ) {
79 override get message(): string {
80 return `Failed to create the workspace search index for '${this.cwd}'.`;
81 }
82 }
83
84 export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedError<WorkspaceSearchIndexScanTimedOut>()(
85 "WorkspaceSearchIndexScanTimedOut",
86 {
87 cwd: Schema.String,
88 timeout: Schema.String,
89 },
90 ) {
91 override get message(): string {
92 return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`;
93 }
94 }
95
96 export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedError<WorkspaceSearchIndexSearchFailed>()(
97 "WorkspaceSearchIndexSearchFailed",
98 {
99 cwd: Schema.String,
100 queryLength: Schema.Number,
101 pageSize: Schema.Number,
102 reason: Schema.String,
103 cause: Schema.optional(Schema.Defect()),
104 },
105 ) {
106 override get message(): string {
107 return `Workspace search failed for '${this.cwd}'.`;
108 }
109 }
110
111 export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedError<WorkspaceSearchIndexRefreshFailed>()(
112 "WorkspaceSearchIndexRefreshFailed",
113 {
114 cwd: Schema.String,
115 reason: Schema.String,
116 cause: Schema.optional(Schema.Defect()),
117 },
118 ) {
119 override get message(): string {
120 return `Failed to refresh the workspace search index for '${this.cwd}'.`;
121 }
122 }
123
124 export class WorkspaceSearchIndexDestroyFailed extends Schema.TaggedError<WorkspaceSearchIndexDestroyFailed>()(
125 "WorkspaceSearchIndexDestroyFailed",
126 {
127 cwd: Schema.String,
128 cause: Schema.Defect(),
129 },
130 ) {
131 override get message(): string {
132 return `Failed to destroy the workspace search index for '${this.cwd}'.`;
133 }
134 }
135
136 export class WorkspaceSearchIndex extends Context.Service<
137 WorkspaceSearchIndex,
138 {
139 readonly list: () => Effect.Effect<ProjectListEntriesResult, WorkspaceSearchIndexSearchFailed>;
140 readonly search: (
141 query: string,
142 limit: number,
143 kind?: ProjectEntryKind,
144 imageOnly?: boolean,
145 ) => Effect.Effect<ProjectSearchEntriesResult, WorkspaceSearchIndexSearchFailed>;
146 readonly searchContents: (
147 input: Omit<ProjectSearchContentsInput, "cwd">,
148 ) => Effect.Effect<
149 ProjectSearchContentsResult,
150 WorkspaceSearchIndexScanTimedOut | WorkspaceSearchIndexSearchFailed
151 >;
152 readonly refresh: () => Effect.Effect<void, WorkspaceSearchIndexRefreshFailed>;
153 }
154 >()("t3/workspace/WorkspaceSearchIndex") {}
155
156 // Only the host separator becomes `/`: on POSIX `\` is a file name character.
157 function toPosixPath(input: string, hostIsWindows: boolean): string {
158 return hostIsWindows ? input.replaceAll("\\", "/") : input;
159 }
160
161 function trimDirectorySeparator(input: string): string {
162 return input.endsWith("/") ? input.slice(0, -1) : input;
163 }
164
165 function parentPathOf(input: string): string | undefined {
166 const separatorIndex = input.lastIndexOf("/");
167 return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex);
168 }
169
170 function toProjectEntry(item: MixedItem, hostIsWindows: boolean): ProjectEntry | null {
171 const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath, hostIsWindows));
172 if (!normalizedPath) {
173 return null;
174 }
175
176 return {
177 path: normalizedPath,
178 kind: item.type,
179 };
180 }
181
182 function toFileEntry(item: FileItem, hostIsWindows: boolean): ProjectEntry | null {
183 const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
184 return normalizedPath ? { path: normalizedPath, kind: "file" } : null;
185 }
186
187 function toDirectoryEntry(item: DirItem, hostIsWindows: boolean): ProjectEntry | null {
188 const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
189 return normalizedPath ? { path: normalizedPath, kind: "directory" } : null;
190 }
191
192 function isLoneFuzzyPath(query: string): boolean {
193 const trimmed = query.trim();
194 return (
195 !/\s/u.test(trimmed) &&
196 (trimmed.startsWith("/") || (trimmed.endsWith("/") && !/^[*!\\]/u.test(trimmed)))
197 );
198 }
199
200 type SearchPage<I> = { readonly items: ReadonlyArray<I>; readonly totalMatched: number };
201
202 function mapFileSearchResult(
203 result: SearchPage<FileItem>,
204 limit: number,
205 hostIsWindows: boolean,
206 imageOnly = false,
207 ): ProjectSearchEntriesResult {
208 const entries = result.items.flatMap((item) => {
209 const entry = toFileEntry(item, hostIsWindows);
210 return entry && (!imageOnly || isProjectFaviconPath(entry.path)) ? [entry] : [];
211 });
212 return {
213 entries: entries.slice(0, limit),
214 truncated: entries.length > limit || result.totalMatched > result.items.length,
215 };
216 }
217
218 function mapDirectorySearchResult(
219 result: SearchPage<DirItem>,
220 limit: number,
221 hostIsWindows: boolean,
222 ): ProjectSearchEntriesResult {
223 const entries = result.items.flatMap((item) => {
224 const entry = toDirectoryEntry(item, hostIsWindows);
225 return entry ? [entry] : [];
226 });
227 const rootDirectoryCount = result.items.some((item) => item.relativePath.length === 0) ? 1 : 0;
228 return {
229 entries: entries.slice(0, limit),
230 truncated: result.totalMatched - rootDirectoryCount > limit,
231 };
232 }
233
234 function mapMixedSearchResult(
235 result: SearchPage<MixedItem>,
236 limit: number,
237 hostIsWindows: boolean,
238 ): { readonly entries: ProjectEntry[]; readonly truncated: boolean } {
239 const entries: ProjectEntry[] = [];
240 for (const item of result.items) {
241 const entry = toProjectEntry(item, hostIsWindows);
242 if (entry) {
243 entries.push(entry);
244 }
245 if (entries.length >= limit) {
246 break;
247 }
248 }
249
250 const rootDirectoryCount = result.items.some(
251 (item) => item.type === "directory" && item.item.relativePath.length === 0,
252 )
253 ? 1
254 : 0;
255 return {
256 entries,
257 truncated: result.totalMatched - rootDirectoryCount > limit,
258 };
259 }
260
261 const WORD_CHARACTER = /[\p{Letter}\p{Mark}\p{Number}_]/u;
262
263 function codePointAt(line: string, index: number): string | undefined {
264 const codePoint = line.codePointAt(index);
265 return codePoint === undefined ? undefined : String.fromCodePoint(codePoint);
266 }
267
268 function codePointBefore(line: string, index: number): string | undefined {
269 if (index <= 0) return undefined;
270 const previousCodeUnit = line.charCodeAt(index - 1);
271 const previousIndex =
272 previousCodeUnit >= 0xdc00 && previousCodeUnit <= 0xdfff ? index - 2 : index - 1;
273 return codePointAt(line, previousIndex);
274 }
275
276 function buildContentSearchQuery(input: Omit<ProjectSearchContentsInput, "cwd">): {
277 readonly searchQuery: string;
278 readonly regexMode: boolean;
279 } {
280 if (input.caseSensitive) {
281 return { searchQuery: input.query, regexMode: input.useRegex };
282 }
283 return input.useRegex
284 ? { searchQuery: `(?i)${input.query}`, regexMode: true }
285 : { searchQuery: input.query.toLowerCase(), regexMode: false };
286 }
287
288 const ASCII_WORD = /^\w+$/;
289
290 // A whole-word search for an ASCII word asks fff for the word itself, so substring hits are dropped
291 // natively instead of filling pages with lines the JS check then discards. `-u` keeps `\b` and `i`
292 // to ASCII: plain search does not fold `ſ` or `K`, and any non-ASCII neighbour stays a candidate.
293 function buildWholeWordPattern(query: string, caseSensitive: boolean): string {
294 return `(?${caseSensitive ? "" : "i"}-u)\\b${query}\\b`;
295 }
296
297 // fff's ranges for that pattern are not used: the JS word check decides, on every hit in the line.
298 function findWordRanges(
299 line: string,
300 word: string,
301 caseSensitive: boolean,
302 ): Array<{ readonly start: number; readonly end: number }> {
303 // No `u` flag: with it, `i` would fold `ſ` and `K` onto ASCII letters.
304 return Array.from(line.matchAll(new RegExp(word, caseSensitive ? "g" : "gi")), (match) => ({
305 start: match.index,
306 end: match.index + match[0].length,
307 }));
308 }
309
310 function mapContentMatchRanges(
311 line: string,
312 byteRanges: ReadonlyArray<readonly [number, number]>,
313 ): Array<{ readonly start: number; readonly end: number }> {
314 const lineBytes = Buffer.from(line);
315 const toStringIndex = (byteOffset: number) => lineBytes.subarray(0, byteOffset).toString().length;
316 return byteRanges.map(([startByte, endByte]) => ({
317 start: toStringIndex(startByte),
318 end: toStringIndex(endByte),
319 }));
320 }
321
322 function escapeRegExp(input: string): string {
323 return input.replace(/[\\^$.*+?()[\]{}|/]/g, "\\$&");
324 }
325
326 // Git before 2.55 reports wrong `--column` values for every match after a line's first, so
327 // later matches are found by their text, searching on from the end of the previous one.
328 function locateRegexMatches(
329 line: string,
330 matches: { readonly column: number; readonly texts: ReadonlyArray<string> } | undefined,
331 ): Array<readonly [number, number]> {
332 if (!matches) return [];
333 const lineBytes = Buffer.from(line);
334 const ranges: Array<readonly [number, number]> = [];
335 let start = matches.column - 1;
336 for (const text of matches.texts) {
337 const textBytes = Buffer.from(text);
338 if (ranges.length > 0) start = lineBytes.indexOf(textBytes, ranges.at(-1)![1]);
339 if (start < 0) break;
340 ranges.push([start, start + textBytes.length]);
341 }
342 return ranges;
343 }
344
345 // `git grep -z` records: NUL-separated fields, one record per line; a truncated last line is dropped.
346 function parseGitGrepRecords(
347 stdout: string,
348 fieldCount: number,
349 truncated: boolean,
350 ): Array<Array<string>> {
351 const lines = stdout.split("\n");
352 if (truncated || lines.at(-1) === "") lines.pop();
353 const records: Array<Array<string>> = [];
354 for (const line of lines) {
355 const fields = line.split("\0");
356 if (fields.length < fieldCount) continue;
357 // Only the last field (the line or match text) can contain NUL bytes.
358 records.push([...fields.slice(0, fieldCount - 1), fields.slice(fieldCount - 1).join("\0")]);
359 }
360 return records;
361 }
362
363 // Post-filter whole words: consuming boundary regexes swallow adjacent separators and widen ranges; \b misses punctuation edges.
364 function isWholeWordRange(
365 line: string,
366 range: { readonly start: number; readonly end: number },
367 ): boolean {
368 if (range.end <= range.start) return false;
369 const isWord = (character: string | undefined) =>
370 character !== undefined && WORD_CHARACTER.test(character);
371 const leftIsBoundary =
372 range.start === 0 ||
373 !isWord(codePointBefore(line, range.start)) ||
374 !isWord(codePointAt(line, range.start));
375 const rightIsBoundary =
376 range.end >= line.length ||
377 !isWord(codePointAt(line, range.end)) ||
378 !isWord(codePointBefore(line, range.end));
379 return leftIsBoundary && rightIsBoundary;
380 }
381
382 function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEntry[] {
383 const entryByPath = new Map(entries.map((entry) => [entry.path, entry]));
384 for (const entry of entries) {
385 let parentPath = parentPathOf(entry.path);
386 while (parentPath) {
387 if (!entryByPath.has(parentPath)) {
388 entryByPath.set(parentPath, { path: parentPath, kind: "directory" });
389 }
390 parentPath = parentPathOf(parentPath);
391 }
392 }
393 return [...entryByPath.values()];
394 }
395
396 interface ScannedDirectory {
397 readonly directory: string;
398 readonly links: ReadonlyArray<ProjectEntry>;
399 /** Every symlink in the folder, including ones dropped for leaving the workspace. */
400 readonly linkPaths: ReadonlyArray<string>;
401 /** A nested repository (or the workspace's own), whose ignore rules own what is below it. */
402 readonly isRepository: boolean;
403 readonly ignoreFile: string | undefined;
404 }
405
406 const joinEntryPath = (directory: string, name: string) =>
407 directory ? `${directory}/${name}` : name;
408
409 /**
410 * fff skips symlinks, and `followSymlinks` drops file links, dedupes folder links, and walks out
411 * of the workspace. So links are read from the folders fff already indexed (ignore rules applied),
412 * without descending into them, and kept only when they resolve inside the workspace.
413 */
414 const scanDirectory = Effect.fn("WorkspaceSearchIndex.scanDirectory")(function* (
415 fileSystem: FileSystem.FileSystem,
416 path: Path.Path,
417 cwd: string,
418 root: string,
419 directory: string,
420 ) {
421 // Effect's readDirectory returns bare names; one lstat per child would cost far more.
422 const children = yield* Effect.promise(() =>
423 NodeFSP.readdir(path.join(cwd, directory), { withFileTypes: true }).catch(() => []),
424 );
425 const links: ProjectEntry[] = [];
426 const linkPaths: string[] = [];
427 let isRepository = false;
428 let ignoreFile: string | undefined;
429 for (const child of children) {
430 if (child.name === ".git") isRepository = true;
431 if (child.name === ".ignore" && child.isFile()) {
432 ignoreFile = yield* fileSystem
433 .readFileString(path.join(cwd, directory, child.name))
434 .pipe(Effect.orElseSucceed(() => undefined));
435 }
436 if (!child.isSymbolicLink()) continue;
437 const entryPath = joinEntryPath(directory, child.name);
438 linkPaths.push(entryPath);
439 const absolutePath = path.join(cwd, entryPath);
440 const target = yield* Effect.all([
441 fileSystem.realPath(absolutePath),
442 fileSystem.stat(absolutePath),
443 ]).pipe(Effect.orElseSucceed(() => undefined));
444 if (!target) continue;
445 const relative = path.relative(root, target[0]);
446 if (
447 relative === ".." ||
448 relative.startsWith(`..${path.sep}`) ||
449 path.isAbsolute(relative) ||
450 relative.split(path.sep).includes(".git") ||
451 (target[1].type !== "Directory" && target[1].type !== "File")
452 ) {
453 continue;
454 }
455 links.push({ path: entryPath, kind: target[1].type === "Directory" ? "directory" : "file" });
456 }
457 return { directory, links, linkPaths, isRepository, ignoreFile } satisfies ScannedDirectory;
458 });
459
460 /** Whether `entryPath` is `directory` itself or below it. */
461 const isWithin = (entryPath: string, directory: string) =>
462 directory === "" || entryPath.startsWith(`${directory}/`);
463
464 /**
465 * fff reads `.ignore` files alongside git's rules, and like ripgrep the nearest one that matches
466 * decides, ahead of any `.gitignore`. `ignoreFiles` is ordered deepest folder first.
467 */
468 function decideByIgnoreFiles(
469 entry: ProjectEntry,
470 ignoreFiles: ReadonlyArray<{ readonly directory: string; readonly file: IgnoreFile }>,
471 ): IgnoreDecision | undefined {
472 for (const { directory, file } of ignoreFiles) {
473 if (!isWithin(entry.path, directory) || entry.path === directory) continue;
474 const relative = directory ? entry.path.slice(directory.length + 1) : entry.path;
475 // Like git and ripgrep, a link is never a folder to a folder-only rule, whatever it targets.
476 const decision = file.decide(relative, false);
477 if (decision) return decision;
478 }
479 return undefined;
480 }
481
482 /** Places each symlink before the first index result that matches the query less closely. */
483 function mergeSymlinkMatches(
484 indexed: ProjectSearchEntriesResult,
485 linkPaths: ReadonlySet<string>,
486 symlinks: ReadonlyArray<ProjectEntry>,
487 query: string,
488 limit: number,
489 ): ProjectSearchEntriesResult {
490 if (linkPaths.size === 0) return indexed;
491 // fff's watcher indexes links created after its scan, under a kind of its own choosing, so a
492 // path it already returned takes the link's kind (or drops out of a kind-filtered search).
493 const symlinkByPath = new Map(symlinks.map((entry) => [entry.path, entry]));
494 const score = (entry: ProjectEntry) =>
495 scoreSubsequenceMatch(entry.path.toLowerCase(), query) ?? Number.POSITIVE_INFINITY;
496 const indexedPaths = new Set(indexed.entries.map((entry) => entry.path));
497 const pending = symlinks
498 .flatMap((entry) => {
499 const entryScore = score(entry);
500 return Number.isFinite(entryScore) && !indexedPaths.has(entry.path)
501 ? [{ entry, score: entryScore }]
502 : [];
503 })
504 .toSorted((left, right) => left.score - right.score);
505 const entries: ProjectEntry[] = [];
506 let nextPending = 0;
507 for (const indexedEntry of indexed.entries) {
508 const entry = linkPaths.has(indexedEntry.path)
509 ? symlinkByPath.get(indexedEntry.path)
510 : indexedEntry;
511 if (!entry) continue;
512 const entryScore = score(entry);
513 while (nextPending < pending.length && pending[nextPending]!.score < entryScore) {
514 entries.push(pending[nextPending++]!.entry);
515 }
516 entries.push(entry);
517 }
518 for (; nextPending < pending.length; nextPending += 1) entries.push(pending[nextPending]!.entry);
519 return {
520 entries: entries.slice(0, limit),
521 truncated: indexed.truncated || entries.length > limit,
522 };
523 }
524
525 // fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file.
526 const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* (
527 vcsProcess: VcsProcess.VcsProcess["Service"],
528 cwd: string,
529 hostIsWindows: boolean,
530 ) {
531 const result = yield* vcsProcess
532 .run({
533 operation: "WorkspaceSearchIndex.listGitIgnoredPaths",
534 command: "git",
535 args: [
536 "-c",
537 "core.fsmonitor=false",
538 "ls-files",
539 "--others",
540 "--ignored",
541 "--exclude-standard",
542 "--directory",
543 "-z",
544 ],
545 cwd,
546 allowNonZeroExit: true,
547 timeoutMs: 10_000,
548 maxOutputBytes: 16 * 1024 * 1024,
549 })
550 .pipe(Effect.orElseSucceed(() => undefined));
551 if (!result || result.exitCode !== 0 || result.stdoutTruncated) return [];
552 // Folders keep their trailing `/`.
553 return result.stdout
554 .split("\0")
555 .filter((entry) => entry.length > 0)
556 .map((entry) => toPosixPath(entry, hostIsWindows));
557 });
558
559 function mixedItemPath(item: MixedItem): string {
560 return item.item.relativePath;
561 }
562
563 function isIgnoredPath(
564 ignoredPaths: ReadonlySet<string>,
565 relativePath: string,
566 hostIsWindows: boolean,
567 ): boolean {
568 if (ignoredPaths.size === 0) return false;
569 let candidate: string | undefined = trimDirectorySeparator(
570 toPosixPath(relativePath, hostIsWindows),
571 );
572 while (candidate) {
573 if (ignoredPaths.has(candidate)) return true;
574 candidate = parentPathOf(candidate);
575 }
576 return false;
577 }
578
579 const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (
580 cwd: string,
581 variant: WorkspaceSearchIndexVariant,
582 ) {
583 const result = yield* Effect.try({
584 try: () =>
585 FileFinder.create({
586 basePath: cwd,
587 disableMmapCache: true,
588 // Content indexing costs CPU/memory; path-only consumers use the lightweight index.
589 disableContentIndexing: variant !== "content",
590 aiMode: false,
591 }),
592 catch: (cause) =>
593 new WorkspaceSearchIndexCreateFailed({
594 cwd,
595 reason: "FileFinder.create threw unexpectedly.",
596 cause,
597 }),
598 });
599 if (result.ok) return result.value;
600 return yield* new WorkspaceSearchIndexCreateFailed({
601 cwd,
602 reason: result.error,
603 });
604 });
605
606 const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(function* <E>(
607 cwd: string,
608 finder: FileFinderType,
609 onFailure: (input: { readonly reason: string; readonly cause?: unknown }) => E,
610 ): Effect.fn.Return<void, E | WorkspaceSearchIndexScanTimedOut> {
611 const result = yield* Effect.tryPromise({
612 try: () => finder.waitForIndexReady(WORKSPACE_INDEX_SCAN_TIMEOUT_MS),
613 catch: (cause) =>
614 onFailure({
615 reason: "FileFinder.waitForIndexReady rejected unexpectedly.",
616 cause,
617 }),
618 });
619 if (!result.ok) {
620 return yield* Effect.fail(onFailure({ reason: result.error }));
621 }
622 if (!result.value) {
623 return yield* new WorkspaceSearchIndexScanTimedOut({
624 cwd,
625 timeout: WORKSPACE_INDEX_SCAN_TIMEOUT,
626 });
627 }
628 });
629
630 export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
631 cwd: string,
632 variant: WorkspaceSearchIndexVariant = "paths",
633 ) {
634 const hostIsWindows = yield* HostProcess.isWindows;
635 const fileSystem = yield* FileSystem.FileSystem;
636 const path = yield* Path.Path;
637 const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) =>
638 Effect.try({
639 try: () => finder.destroy(),
640 catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }),
641 }).pipe(Effect.orDie),
642 );
643 let initialScanTimedOut = false;
644 // The content index reads every file, which takes minutes where antivirus scans each open,
645 // so content searches do not wait for it; git grep answers until it is ready.
646 let contentIndexPending = variant === "content";
647 const vcsProcess = yield* VcsProcess.VcsProcess;
648
649 /** The nearest `.ignore` with a pattern matching `relativePath` itself decides. */
650 const makeIgnoreFileDecider = () => {
651 const ignoreFiles = new Map<string, IgnoreFile | undefined>();
652 const readIgnoreFile = Effect.fn("WorkspaceSearchIndex.readIgnoreFile")(function* (
653 directory: string,
654 ) {
655 if (ignoreFiles.has(directory)) return ignoreFiles.get(directory);
656 const contents = yield* fileSystem
657 .readFileString(path.join(cwd, directory, ".ignore"))
658 .pipe(Effect.option);
659 const file = Option.isSome(contents) ? parseIgnoreFile(contents.value) : undefined;
660 ignoreFiles.set(directory, file);
661 return file;
662 });
663 return Effect.fn("WorkspaceSearchIndex.decideByIgnoreFile")(function* (
664 relativePath: string,
665 isFolder: boolean,
666 ) {
667 const segments = relativePath.split("/");
668 for (let ancestor = segments.length - 1; ancestor >= 0; ancestor -= 1) {
669 const directory = segments.slice(0, ancestor).join("/");
670 const file = yield* readIgnoreFile(directory);
671 const relative = directory ? relativePath.slice(directory.length + 1) : relativePath;
672 const decision = file?.decide(relative, isFolder);
673 if (decision) return decision;
674 }
675 return undefined;
676 });
677 };
678
679 // Like ripgrep and fff, a `.ignore` that re-includes a path overrules Git's ignore rules for it.
680 const loadIgnoredPaths = Effect.fn("WorkspaceSearchIndex.loadIgnoredPaths")(function* () {
681 const decide = makeIgnoreFileDecider();
682 const ignored = new Set<string>();
683 for (const entry of yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows)) {
684 const relativePath = trimDirectorySeparator(entry);
685 if ((yield* decide(relativePath, entry.endsWith("/"))) !== "included") {
686 ignored.add(relativePath);
687 }
688 }
689 return ignored;
690 });
691 let ignoredPaths = yield* loadIgnoredPaths();
692 if (variant === "paths") {
693 yield* waitForIndexReady(
694 cwd,
695 finder,
696 ({ reason, cause }) =>
697 new WorkspaceSearchIndexCreateFailed({
698 cwd,
699 reason,
700 cause,
701 }),
702 ).pipe(
703 Effect.catchTags({
704 WorkspaceSearchIndexScanTimedOut: () =>
705 Effect.sync(() => {
706 initialScanTimedOut = true;
707 }),
708 }),
709 );
710 }
711
712 // fff's isScanning takes the picker lock too, so progress is read off the JS thread.
713 const isScanning = Effect.promise(async () => {
714 try {
715 const progress = await finder.getScanProgressAsync();
716 return !progress.ok || progress.value.isScanning;
717 } catch {
718 return true;
719 }
720 });
721 const hasIncompleteInitialScan = () => (initialScanTimedOut ? isScanning : Effect.succeed(false));
722
723 const isContentIndexReady = Effect.fn("WorkspaceSearchIndex.isContentIndexReady")(function* () {
724 if (!contentIndexPending) return true;
725 const progress = yield* Effect.promise(() =>
726 finder.getScanProgressAsync().catch(() => undefined),
727 );
728 if (progress?.ok && !progress.value.isScanning && progress.value.isWarmupComplete) {
729 contentIndexPending = false;
730 }
731 return !contentIndexPending;
732 });
733
734 const runGitGrep = Effect.fn("WorkspaceSearchIndex.runGitGrep")(function* (
735 pattern: string,
736 options: {
737 readonly regexMode: boolean;
738 readonly ignoreCase: boolean;
739 readonly maxMatchesPerFile: number;
740 readonly onlyMatching: boolean;
741 },
742 ) {
743 return yield* vcsProcess
744 .run({
745 operation: "WorkspaceSearchIndex.gitGrep",
746 command: "git",
747 args: [
748 "-c",
749 "core.quotePath=false",
750 "-c",
751 "core.fsmonitor=false",
752 "grep",
753 "--untracked",
754 "--no-color",
755 "-I",
756 "-n",
757 "-z",
758 "-m",
759 String(options.maxMatchesPerFile),
760 ...(options.onlyMatching ? ["-o", "--column"] : []),
761 ...(options.ignoreCase ? ["-i"] : []),
762 options.regexMode ? "-P" : "-F",
763 "-e",
764 pattern,
765 ],
766 cwd,
767 allowNonZeroExit: true,
768 // Defender scanning fff's warmup reads slows git too; a late answer beats a failure.
769 timeoutMs: 30_000,
770 maxOutputBytes: GIT_GREP_MAX_OUTPUT_BYTES,
771 outputMode: "truncate",
772 appendTruncationMarker: false,
773 })
774 .pipe(Effect.option);
775 });
776
777 // fff leaves out what `.ignore` files exclude, which git cannot evaluate; the nearest
778 // `.ignore` that matches a path or one of its folders decides, as in decideByIgnoreFiles.
779 const makeIgnoreFileFilter = () => {
780 const decide = makeIgnoreFileDecider();
781 return Effect.fn("WorkspaceSearchIndex.isExcludedByIgnoreFiles")(function* (
782 relativePath: string,
783 ) {
784 const segments = relativePath.split("/");
785 for (let depth = 1; depth <= segments.length; depth += 1) {
786 const candidate = segments.slice(0, depth).join("/");
787 if ((yield* decide(candidate, depth < segments.length)) === "ignored") return true;
788 }
789 return false;
790 });
791 };
792
793 // fff's grep before its content index is ready opens every file itself, as slowly as the build.
794 // None when git cannot search this folder (not a repository, git missing, no PCRE for a regex).
795 const searchContentsWithGit = Effect.fn("WorkspaceSearchIndex.searchContentsWithGit")(function* (
796 input: Omit<ProjectSearchContentsInput, "cwd">,
797 ): Effect.fn.Return<Option.Option<ProjectSearchContentsResult>> {
798 const options = {
799 ignoreCase: !input.caseSensitive,
800 maxMatchesPerFile: CONTENT_SEARCH_MAX_MATCHES_PER_FILE,
801 onlyMatching: false,
802 };
803 let regexMode = input.useRegex;
804 let regexFallbackError: string | undefined;
805 let result = yield* runGitGrep(input.query, { ...options, regexMode });
806 if (regexMode && Option.isSome(result) && result.value.exitCode === 128) {
807 if (/USE_LIBPCRE|Perl-compatible/i.test(result.value.stderr)) return Option.none();
808 // fff searches an invalid pattern as a literal and reports why.
809 regexFallbackError = result.value.stderr.trim().split("\n")[0] || "Invalid regex";
810 regexMode = false;
811 result = yield* runGitGrep(input.query, { ...options, regexMode });
812 }
813 if (Option.isNone(result) || result.value.exitCode > 1) return Option.none();
814
815 // Per line: the first match's byte column and every match's text, in order.
816 const regexMatches = new Map<string, { readonly column: number; readonly texts: string[] }>();
817 let rangesTruncated = false;
818 if (regexMode) {
819 // Ranges come from git too: running the user's pattern in JS could backtrack without bound.
820 const ranges = yield* runGitGrep(input.query, { ...options, regexMode, onlyMatching: true });
821 // Without ranges, whole-word filtering would drop every hit and report no matches.
822 if (Option.isNone(ranges) || ranges.value.exitCode > 1) return Option.none();
823 rangesTruncated = ranges.value.stdoutTruncated;
824 if (ranges.value.exitCode === 0) {
825 for (const record of parseGitGrepRecords(
826 ranges.value.stdout,
827 4,
828 ranges.value.stdoutTruncated,
829 )) {
830 const [relativePath, lineNumber, column, text] = record as [
831 string,
832 string,
833 string,
834 string,
835 ];
836 const key = `${relativePath}\0${lineNumber}`;
837 const lineMatches = regexMatches.get(key);
838 if (lineMatches) lineMatches.texts.push(text);
839 else regexMatches.set(key, { column: Number(column), texts: [text] });
840 }
841 }
842 }
843 const literal = regexMode
844 ? undefined
845 : new RegExp(escapeRegExp(input.query), input.caseSensitive ? "gu" : "giu");
846
847 const isExcludedByIgnoreFiles = makeIgnoreFileFilter();
848 const matches: Array<ProjectSearchContentsResult["matches"][number]> = [];
849 let rangesCutOff = false;
850 for (const record of parseGitGrepRecords(
851 result.value.stdout,
852 3,
853 result.value.stdoutTruncated,
854 )) {
855 const [relativePath, lineNumber, rawLine] = record as [string, string, string];
856 if (yield* isExcludedByIgnoreFiles(relativePath)) continue;
857 const key = `${relativePath}\0${lineNumber}`;
858 // Lines past the cut-off of a truncated ranges run have no ranges; stop there instead.
859 if (rangesTruncated && !regexMatches.has(key)) {
860 rangesCutOff = true;
861 break;
862 }
863 const lineContent = (rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine).slice(
864 0,
865 CONTENT_LINE_MAX_LENGTH,
866 );
867 const matchRanges = (
868 literal
869 ? Array.from(lineContent.matchAll(literal), (match) => ({
870 start: match.index,
871 end: match.index + match[0].length,
872 }))
873 : mapContentMatchRanges(
874 lineContent,
875 locateRegexMatches(lineContent, regexMatches.get(key)),
876 )
877 ).filter(
878 (range) =>
879 range.end <= lineContent.length &&
880 (!input.wholeWord || isWholeWordRange(lineContent, range)),
881 );
882 if (input.wholeWord && matchRanges.length === 0) continue;
883 matches.push({
884 path: relativePath,
885 lineNumber: Number(lineNumber),
886 lineContent,
887 matchRanges,
888 });
889 if (matches.length > input.limit) break;
890 }
891 return Option.some({
892 matches: matches.slice(0, input.limit),
893 truncated: matches.length > input.limit || result.value.stdoutTruncated || rangesCutOff,
894 ...(regexFallbackError !== undefined ? { regexFallbackError } : {}),
895 });
896 });
897
898 const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* <A>(
899 query: string,
900 pageSize: number,
901 operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch",
902 execute: () => Promise<Result<A>>,
903 ): Effect.fn.Return<A, WorkspaceSearchIndexSearchFailed> {
904 // fff's searches wait on its picker lock, which its watcher can hold for minutes.
905 const result = yield* Effect.tryPromise({
906 try: execute,
907 catch: (cause) =>
908 new WorkspaceSearchIndexSearchFailed({
909 cwd,
910 queryLength: query.length,
911 pageSize,
912 reason: `FileFinder.${operation} rejected unexpectedly.`,
913 cause,
914 }),
915 });
916 if (!result.ok) {
917 return yield* new WorkspaceSearchIndexSearchFailed({
918 cwd,
919 queryLength: query.length,
920 pageSize,
921 reason: result.error,
922 });
923 }
924 return result.value;
925 });
926
927 const root = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => cwd));
928
929 const listIndexedDirectories = Effect.fn("WorkspaceSearchIndex.listIndexedDirectories")(
930 function* () {
931 const directories: string[] = [];
932 // fff's pageIndex is an offset into the results, not a page number.
933 for (let pageIndex = 0; ; pageIndex += SYMLINK_DIRECTORY_PAGE_SIZE) {
934 const result = yield* Effect.promise(async () => {
935 try {
936 return await finder.directorySearchAsync("", {
937 pageIndex,
938 pageSize: SYMLINK_DIRECTORY_PAGE_SIZE,
939 });
940 } catch {
941 return undefined;
942 }
943 });
944 if (!result?.ok) return directories;
945 for (const item of result.value.items) {
946 const directory = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
947 if (directory) directories.push(directory);
948 }
949 if (result.value.items.length < SYMLINK_DIRECTORY_PAGE_SIZE) return directories;
950 }
951 },
952 );
953
954 /** Ignore rules belong to the repository that owns a path, so each one answers for its own. */
955 const listGitIgnored = Effect.fn("WorkspaceSearchIndex.listGitIgnored")(function* (
956 repository: string,
957 entries: ReadonlyArray<ProjectEntry>,
958 ) {
959 const relativePaths = entries.map((entry) =>
960 repository ? entry.path.slice(repository.length + 1) : entry.path,
961 );
962 const repositoryCwd = path.join(cwd, repository);
963 const result = yield* vcsProcess
964 .run({
965 operation: "WorkspaceSearchIndex.listGitIgnored",
966 command: "git",
967 // fff applies ignore rules to tracked paths too, so the index is not consulted.
968 args: ["-c", "core.fsmonitor=false", "check-ignore", "--no-index", "-z", "--stdin"],
969 cwd: repositoryCwd,
970 stdin: `${relativePaths.join("\0")}\0`,
971 allowNonZeroExit: true,
972 timeoutMs: 10_000,
973 maxOutputBytes: 16 * 1024 * 1024,
974 })
975 .pipe(Effect.orElseSucceed(() => undefined));
976 // Exit 1 means nothing matched.
977 if (result?.exitCode === 1) return [];
978 if (result?.exitCode !== 0 || result.stdoutTruncated) {
979 // Outside git no rules apply (git exits before reading stdin, so the cause is lost).
980 // Any other failure hides the repository's links rather than leaking ones it ignores.
981 const insideRepository = yield* vcsProcess
982 .run({
983 operation: "WorkspaceSearchIndex.listGitIgnored",
984 command: "git",
985 args: ["rev-parse", "--git-dir"],
986 cwd: repositoryCwd,
987 timeoutMs: 10_000,
988 })
989 .pipe(
990 Effect.as(true),
991 Effect.orElseSucceed(() => false),
992 );
993 return insideRepository ? entries.map((entry) => entry.path) : [];
994 }
995 return result.stdout
996 .split("\0")
997 .filter(Boolean)
998 .map((relativePath) => (repository ? `${repository}/${relativePath}` : relativePath));
999 });
1000
1001 const scanSymlinkEntries = Effect.fn("WorkspaceSearchIndex.scanSymlinkEntries")(function* () {
1002 const indexComplete = !(yield* isScanning);
1003 const directories = yield* listIndexedDirectories();
1004 const scanned = yield* Effect.forEach(
1005 ["", ...directories],
1006 (directory) => scanDirectory(fileSystem, path, cwd, root, directory),
1007 { concurrency: SYMLINK_SCAN_CONCURRENCY },
1008 );
1009 const candidates = scanned.flatMap((directory) => directory.links);
1010 const linkPaths = new Set(scanned.flatMap((directory) => directory.linkPaths));
1011 if (candidates.length === 0) return { entries: candidates, linkPaths, indexComplete };
1012
1013 const ignoreFiles = scanned
1014 .flatMap(({ directory, ignoreFile }) =>
1015 ignoreFile === undefined ? [] : [{ directory, file: parseIgnoreFile(ignoreFile) }],
1016 )
1017 .toSorted((left, right) => right.directory.length - left.directory.length);
1018 const decisions = new Map(
1019 candidates.map((entry) => [entry.path, decideByIgnoreFiles(entry, ignoreFiles)]),
1020 );
1021 const repositories = [
1022 "",
1023 ...scanned.flatMap(({ directory, isRepository }) =>
1024 isRepository && directory ? [directory] : [],
1025 ),
1026 ].toSorted((left, right) => right.length - left.length);
1027 const entriesByRepository = Map.groupBy(
1028 candidates.filter((entry) => decisions.get(entry.path) === undefined),
1029 (entry) =>
1030 repositories.find(
1031 (repository) => isWithin(entry.path, repository) && entry.path !== repository,
1032 ) ?? "",
1033 );
1034 const ignored = new Set(
1035 (yield* Effect.forEach(
1036 entriesByRepository,
1037 ([repository, entries]) => listGitIgnored(repository, entries),
1038 { concurrency: 4 },
1039 )).flat(),
1040 );
1041 return {
1042 entries: candidates.filter((entry) =>
1043 decisions.get(entry.path) === undefined
1044 ? !ignored.has(entry.path)
1045 : decisions.get(entry.path) === "included",
1046 ),
1047 linkPaths,
1048 indexComplete,
1049 };
1050 });
1051
1052 // One scan runs at a time and outlives the reads that started it. Only the first read waits for
1053 // it in full: after that a change or expiry marks the result stale, and reads wait briefly for
1054 // the rescan before answering from the last finished one, so file churn cannot stall search.
1055 type SymlinkScan = Effect.Success<ReturnType<typeof scanSymlinkEntries>>;
1056 let lastSymlinkScan: SymlinkScan | undefined;
1057 let lastSymlinkScanExpiresAt: number | undefined;
1058 let pendingSymlinkScan: Deferred.Deferred<SymlinkScan> | undefined;
1059 let symlinkScanStale = false;
1060 let knownLinkPaths: ReadonlySet<string> = new Set();
1061 const invalidateSymlinkScan = () => {
1062 symlinkScanStale = true;
1063 };
1064 const isKnownLink = (absolutePath: string) =>
1065 [cwd, root].some((base) =>
1066 knownLinkPaths.has(toPosixPath(path.relative(base, absolutePath), hostIsWindows)),
1067 );
1068 const onWatchBatch = (events: ReadonlyArray<WatchEvent>) => {
1069 // fff reports a link replaced in place as `modified`, so edits to known links count too.
1070 const editsOnly = events.every(
1071 (event) =>
1072 event.kind === "modified" &&
1073 !/(^|[\\/])\.(git)?ignore$/.test(event.path) &&
1074 !isKnownLink(event.path),
1075 );
1076 if (!editsOnly) invalidateSymlinkScan();
1077 };
1078 // fff refuses subscriptions until its watcher is up, which can trail the initial scan, so
1079 // each read retries until one is in place. Subscribing waits on fff's picker lock, which its
1080 // watcher holds while it indexes a new folder, so it runs off the JS thread.
1081 let unwatch: WatchUnsubscribe | undefined;
1082 yield* Effect.addFinalizer(() => Effect.sync(() => unwatch?.()));
1083 const ensureWatching = Effect.fn("WorkspaceSearchIndex.ensureWatching")(function* () {
1084 if (unwatch) return true;
1085 const result = yield* Effect.promise(() =>
1086 finder.watchAsync(onWatchBatch).catch(() => undefined),
1087 );
1088 if (result?.ok) unwatch = result.value;
1089 return unwatch !== undefined;
1090 });
1091 const indexScope = yield* Effect.scope;
1092 const startSymlinkScan = Effect.fn("WorkspaceSearchIndex.startSymlinkScan")(function* () {
1093 symlinkScanStale = false;
1094 const done = yield* Deferred.make<SymlinkScan>();
1095 pendingSymlinkScan = done;
1096 yield* Effect.gen(function* () {
1097 // Subscribing before the scan starts means no change during it goes unseen.
1098 const watched = yield* ensureWatching();
1099 const result = yield* scanSymlinkEntries();
1100 lastSymlinkScan = result;
1101 knownLinkPaths = result.linkPaths;
1102 // Without a watcher, or before fff has seen every folder, the result is kept only briefly.
1103 lastSymlinkScanExpiresAt =
1104 watched && result.indexComplete
1105 ? undefined
1106 : (yield* Clock.currentTimeMillis) + UNWATCHED_SYMLINK_SCAN_TTL_MS;
1107 return result;
1108 }).pipe(
1109 Effect.onExit((exit) =>
1110 Effect.sync(() => {
1111 pendingSymlinkScan = undefined;
1112 }).pipe(Effect.andThen(Deferred.done(done, exit))),
1113 ),
1114 Effect.forkIn(indexScope),
1115 );
1116 return done;
1117 });
1118 const listSymlinkEntries = Effect.fn("WorkspaceSearchIndex.listSymlinkEntries")(function* () {
1119 if (
1120 lastSymlinkScanExpiresAt !== undefined &&
1121 (yield* Clock.currentTimeMillis) >= lastSymlinkScanExpiresAt
1122 ) {
1123 symlinkScanStale = true;
1124 }
1125 const pending =
1126 pendingSymlinkScan ??
1127 (lastSymlinkScan === undefined || symlinkScanStale ? yield* startSymlinkScan() : undefined);
1128 let result = lastSymlinkScan;
1129 if (pending && result === undefined) {
1130 result = yield* Deferred.await(pending);
1131 } else if (pending) {
1132 const fresh = yield* Deferred.await(pending).pipe(Effect.timeoutOption(SYMLINK_RESCAN_WAIT));
1133 if (Option.isSome(fresh)) result = fresh.value;
1134 }
1135 if (result === undefined)
1136 return yield* Effect.die("The symlink scan finished without a result.");
1137 return {
1138 // Git's ignore rules hide a link the same way searchVisible hides the files it indexes.
1139 entries: result.entries.filter(
1140 (entry) => !isIgnoredPath(ignoredPaths, entry.path, hostIsWindows),
1141 ),
1142 linkPaths: result.linkPaths,
1143 };
1144 });
1145
1146 // Grows the first page until hidden entries can no longer crowd visible ones out of it.
1147 const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* <I, Metadata>(
1148 query: string,
1149 pageSize: number,
1150 operation: "directorySearch" | "fileSearch" | "mixedSearch",
1151 execute: (pageSize: number) => Promise<Result<SearchPage<I> & Metadata>>,
1152 pathOf: (item: I) => string,
1153 ): Effect.fn.Return<SearchPage<I> & Metadata, WorkspaceSearchIndexSearchFailed> {
1154 let size = pageSize;
1155 while (true) {
1156 const fetchSize = size;
1157 const result = yield* runSearch(query, fetchSize, operation, () => execute(fetchSize));
1158 if (ignoredPaths.size === 0) return result;
1159 const items = result.items.filter(
1160 (item) => !isIgnoredPath(ignoredPaths, pathOf(item), hostIsWindows),
1161 );
1162 if (
1163 items.length >= pageSize ||
1164 result.items.length >= result.totalMatched ||
1165 fetchSize >= WORKSPACE_INDEX_PAGE_SIZE
1166 ) {
1167 return {
1168 ...result,
1169 items,
1170 totalMatched: result.totalMatched - (result.items.length - items.length),
1171 };
1172 }
1173 size = Math.min(WORKSPACE_INDEX_PAGE_SIZE, fetchSize * 4);
1174 }
1175 });
1176
1177 const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
1178 "WorkspaceSearchIndex.refresh",
1179 )(function* () {
1180 // scanFiles waits on fff's picker lock, which its watcher can hold for seconds.
1181 const result = yield* Effect.tryPromise({
1182 try: () => finder.scanFilesAsync(),
1183 catch: (cause) =>
1184 new WorkspaceSearchIndexRefreshFailed({
1185 cwd,
1186 reason: "FileFinder.scanFilesAsync rejected unexpectedly.",
1187 cause,
1188 }),
1189 });
1190 if (!result.ok) {
1191 return yield* new WorkspaceSearchIndexRefreshFailed({
1192 cwd,
1193 reason: result.error,
1194 });
1195 }
1196 // A rescan that outlasts the wait keeps running, and the finder answers
1197 // from its previous index until it finishes, so the index stays usable.
1198 yield* waitForIndexReady(
1199 cwd,
1200 finder,
1201 ({ reason, cause }) =>
1202 new WorkspaceSearchIndexRefreshFailed({
1203 cwd,
1204 reason,
1205 cause,
1206 }),
1207 ).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void }));
1208 ignoredPaths = yield* loadIgnoredPaths();
1209 invalidateSymlinkScan();
1210 });
1211
1212 const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
1213 function* () {
1214 const incompleteBeforeQuery = yield* hasIncompleteInitialScan();
1215 const result = yield* searchVisible(
1216 "",
1217 WORKSPACE_INDEX_PAGE_SIZE,
1218 "mixedSearch",
1219 (pageSize) => finder.mixedSearchAsync("", { pageSize }),
1220 mixedItemPath,
1221 );
1222 const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES, hostIsWindows);
1223 const symlinks = yield* listSymlinkEntries();
1224 // fff's watcher can index a link the scan rejected; only accepted links are listed.
1225 const indexed = mapped.entries.filter((entry) => !symlinks.linkPaths.has(entry.path));
1226 const sortedEntries = withDirectoryAncestors([...indexed, ...symlinks.entries]).toSorted(
1227 (left, right) => left.path.localeCompare(right.path),
1228 );
1229 const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES);
1230 return {
1231 entries,
1232 truncated:
1233 incompleteBeforeQuery || mapped.truncated || entries.length < sortedEntries.length,
1234 };
1235 },
1236 );
1237
1238 const searchNames = Effect.fn("WorkspaceSearchIndex.searchNames")(function* (
1239 query: string,
1240 limit: number,
1241 kind: ProjectEntryKind | undefined,
1242 imageOnly: boolean | undefined,
1243 ) {
1244 const { entries: allSymlinks, linkPaths } = yield* listSymlinkEntries();
1245 const symlinks = allSymlinks.filter((entry) =>
1246 imageOnly
1247 ? entry.kind === "file" && isProjectFaviconPath(entry.path)
1248 : kind === undefined || entry.kind === kind,
1249 );
1250 // Index results that are filtered-out links get dropped in the merge, so fetch past them.
1251 const indexLimit = limit + linkPaths.size - symlinks.length;
1252 const pageSize = Math.max(1, indexLimit + 1);
1253 const merge = (indexed: ProjectSearchEntriesResult) =>
1254 mergeSymlinkMatches(indexed, linkPaths, symlinks, query, limit);
1255 if (kind === "file" || imageOnly) {
1256 const searchFiles = (fileQuery: string, size: number) =>
1257 searchVisible(
1258 query,
1259 size,
1260 "fileSearch",
1261 (fetchSize) => finder.fileSearchAsync(fileQuery, { pageSize: fetchSize }),
1262 (item: FileItem) => item.relativePath,
1263 );
1264 // Adding a constraint changes fff's lone-path parse; offset pages can skip tied matches.
1265 if (imageOnly && isLoneFuzzyPath(query)) {
1266 const result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE);
1267 return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, true));
1268 }
1269 let result = yield* searchFiles(
1270 imageOnly ? `${query} ${IMAGE_SEARCH_CONSTRAINT}` : query,
1271 pageSize,
1272 );
1273 if (
1274 imageOnly &&
1275 query.trim() &&
1276 result.totalFiles > FFF_FILENAME_FALLBACK_MAX_MATCHES &&
1277 result.totalMatched > 0
1278 ) {
1279 const unfiltered = yield* runSearch(query, 1, "fileSearch", () =>
1280 finder.fileSearchAsync(query, { pageSize: 1 }),
1281 );
1282 if (unfiltered.totalMatched > FFF_FILENAME_FALLBACK_MAX_MATCHES) {
1283 result = yield* searchFiles(query, WORKSPACE_INDEX_PAGE_SIZE);
1284 }
1285 }
1286 return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, imageOnly));
1287 }
1288 if (kind === "directory") {
1289 const result = yield* searchVisible(
1290 query,
1291 pageSize,
1292 "directorySearch",
1293 (size) => finder.directorySearchAsync(query, { pageSize: size }),
1294 (item) => item.relativePath,
1295 );
1296 return merge(mapDirectorySearchResult(result, indexLimit, hostIsWindows));
1297 }
1298 const result = yield* searchVisible(
1299 query,
1300 pageSize,
1301 "mixedSearch",
1302 (size) => finder.mixedSearchAsync(query, { pageSize: size }),
1303 mixedItemPath,
1304 );
1305 return merge(mapMixedSearchResult(result, indexLimit, hostIsWindows));
1306 });
1307
1308 const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn(
1309 "WorkspaceSearchIndex.search",
1310 )(function* (rawQuery, limit, kind, imageOnly) {
1311 // fff parses any word containing "[" or "{" as a glob and has no escape for them; fuzzy
1312 // matching is subsequence-based, so dropping them still matches "[id]" and "a{b}" paths.
1313 // "{" is dropped only when the query finds nothing as a glob, so "*.{ts,tsx}" keeps working.
1314 const query = rawQuery.replaceAll("[", "");
1315 const incompleteBeforeQuery = yield* hasIncompleteInitialScan();
1316 let mapped = yield* searchNames(query, limit, kind, imageOnly);
1317 if (mapped.entries.length === 0 && query.includes("{")) {
1318 mapped = yield* searchNames(query.replaceAll("{", ""), limit, kind, imageOnly);
1319 }
1320 return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated };
1321 });
1322
1323 const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn(
1324 "WorkspaceSearchIndex.searchContents",
1325 )(function* (input) {
1326 if (!(yield* isContentIndexReady())) {
1327 const gitResult = yield* searchContentsWithGit(input);
1328 if (Option.isSome(gitResult)) return gitResult.value;
1329 yield* waitForIndexReady(
1330 cwd,
1331 finder,
1332 ({ reason, cause }) =>
1333 new WorkspaceSearchIndexSearchFailed({
1334 cwd,
1335 queryLength: input.query.length,
1336 pageSize: input.limit,
1337 reason,
1338 cause,
1339 }),
1340 );
1341 contentIndexPending = false;
1342 }
1343 // fff reads glob and path syntax out of a query even in regex mode, so only a bare word qualifies.
1344 const wholeWordPrefilter = input.wholeWord && !input.useRegex && ASCII_WORD.test(input.query);
1345 let { searchQuery, regexMode } = wholeWordPrefilter
1346 ? { searchQuery: buildWholeWordPattern(input.query, input.caseSensitive), regexMode: true }
1347 : buildContentSearchQuery(input);
1348 const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS;
1349 // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing.
1350 const rawPageSize = input.wholeWord
1351 ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE)
1352 : input.limit;
1353 const matches: Array<ProjectSearchContentsResult["matches"][number]> = [];
1354 let nextCursor: GrepCursor | null = null;
1355 let regexFallbackError: string | undefined;
1356 let retryLiterally = false;
1357
1358 do {
1359 const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now()));
1360 const result = yield* runSearch(input.query, input.limit, "grep", () =>
1361 finder.grepAsync(searchQuery, {
1362 mode: regexMode ? "regex" : "plain",
1363 smartCase: !input.caseSensitive && !regexMode,
1364 maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize),
1365 pageSize: rawPageSize,
1366 cursor: nextCursor,
1367 timeBudgetMs: remainingTimeBudgetMs,
1368 }),
1369 );
1370 // fff searches an invalid pattern as a literal, which would keep the `(?i)` prefix.
1371 retryLiterally = regexMode && !input.caseSensitive && result.regexFallbackError !== undefined;
1372 if (retryLiterally) {
1373 regexFallbackError = result.regexFallbackError;
1374 ({ searchQuery, regexMode } = buildContentSearchQuery({ ...input, useRegex: false }));
1375 continue;
1376 }
1377
1378 for (const match of result.items) {
1379 if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue;
1380 const matchRanges = (
1381 wholeWordPrefilter
1382 ? findWordRanges(match.lineContent, input.query, input.caseSensitive)
1383 : mapContentMatchRanges(match.lineContent, match.matchRanges)
1384 ).filter((range) => !input.wholeWord || isWholeWordRange(match.lineContent, range));
1385 // fff reports hits past its 512-char lineContent cap with no ranges, so whole words can't be verified there.
1386 if (input.wholeWord && matchRanges.length === 0) continue;
1387 matches.push({
1388 path: toPosixPath(match.relativePath, hostIsWindows),
1389 lineNumber: match.lineNumber,
1390 lineContent: match.lineContent,
1391 matchRanges,
1392 });
1393 }
1394 nextCursor = result.nextCursor;
1395 regexFallbackError ??= result.regexFallbackError;
1396 } while (
1397 retryLiterally ||
1398 (matches.length < input.limit && nextCursor !== null && performance.now() < deadline)
1399 );
1400
1401 return {
1402 matches: matches.slice(0, input.limit),
1403 truncated: matches.length > input.limit || nextCursor !== null,
1404 ...(regexFallbackError !== undefined ? { regexFallbackError } : {}),
1405 };
1406 });
1407
1408 return WorkspaceSearchIndex.of({ list, refresh, search, searchContents });
1409 });
1410
1411 export const WORKSPACE_SEARCH_INDEX_VARIANTS = ["paths", "content"] as const;
1412 export type WorkspaceSearchIndexVariant = (typeof WORKSPACE_SEARCH_INDEX_VARIANTS)[number];
1413
1414 export const workspaceSearchIndexKey = (cwd: string, variant: WorkspaceSearchIndexVariant) =>
1415 `${variant}\n${cwd}`;
1416
1417 function parseWorkspaceSearchIndexKey(key: string): {
1418 readonly cwd: string;
1419 readonly variant: WorkspaceSearchIndexVariant;
1420 } {
1421 const separatorIndex = key.indexOf("\n");
1422 return {
1423 variant: key.slice(0, separatorIndex) as WorkspaceSearchIndexVariant,
1424 cwd: key.slice(separatorIndex + 1),
1425 };
1426 }
1427
1428 /** @public */
1429 export const layer = (key: string) => {
1430 const { cwd, variant } = parseWorkspaceSearchIndexKey(key);
1431 return Layer.effect(WorkspaceSearchIndex, make(cwd, variant));
1432 };
1433
1434 export class WorkspaceSearchIndexMap extends LayerMap.Service<WorkspaceSearchIndexMap>()(
1435 "t3/workspace/WorkspaceSearchIndexMap",
1436 {
1437 lookup: layer,
1438 idleTimeToLive: WORKSPACE_INDEX_IDLE_TTL,
1439 },
1440 ) {}