From db304bf4bddd99f6acc4826888678f6b6bbde98e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 19:03:03 +0000 Subject: [PATCH] fix(server): whole-word content searches no longer miss matching files A whole-word search ran as a plain substring search and dropped the non-word hits in JS. For a word that is mostly a substring elsewhere ("port" in "import", "export"), the 250 ms search deadline ran out on discarded hits and the result listed only a few of the matching files. A plain ASCII word now goes to fff as an ASCII word-boundary regex, so only whole-word lines come back. The JS word check still decides each range, case folding stays ASCII-only as in plain search, and the overall deadline is unchanged. Co-authored-by: Lakshmi Tanmay Co-authored-by: Julius Marminge Co-Authored-By: Claude Fable 5.2 --- .../workspace/WorkspaceSearchIndex.test.ts | 43 +++++++++++++++++++ .../src/workspace/WorkspaceSearchIndex.ts | 36 ++++++++++++++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 265a72a98a4aa..421f36df73f68 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -290,6 +290,8 @@ it.effect("continues whole-word searches after a filtered grep page", () => truncated: false, }); expect(grep).toHaveBeenCalledTimes(2); + expect(grep.mock.calls[0]?.[0]).toBe(String.raw`(?-u)\bneedle\b`); + expect(grep.mock.calls[0]?.[1]?.mode).toBe("regex"); expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor); }), ).pipe(Effect.provide(testLayer)), @@ -772,3 +774,44 @@ it.live("finds Git-ignored files that a root .ignore re-includes", () => expect(result.entries).toEqual([{ kind: "file", path: "notes/daily/standup.md" }]); }).pipe(Effect.scoped, Effect.provide(testLayer)), ); + +// Real fff: the native whole-word pattern must return what the plain search and the JS word check did. +it.live.each([ + { query: "needle", caseSensitive: true, expected: ["notes.md:1:0-6", "words.ts:2:0-6,8-14"] }, + { query: "NEEDLE", caseSensitive: false, expected: ["notes.md:1:0-6", "words.ts:2:0-6,8-14"] }, + // fff reads `*.ts` as a file constraint, which a regex built around the whole query would lose. + { query: "*.ts needle", caseSensitive: true, expected: ["words.ts:2:0-6,8-14"] }, + // Plain search folds ASCII case only: `ſ` is not an `s`, and the Kelvin sign is not a `k`. + { query: "s", caseSensitive: false, expected: ["fold.txt:2:0-1"] }, + { query: "k", caseSensitive: false, expected: ["fold.txt:4:0-1"] }, +])("matches whole words in a real workspace for $query", ({ query, caseSensitive, expected }) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-search-whole-word-" }); + yield* fileSystem.writeFileString( + path.join(cwd, "words.ts"), + "needleSuffix\nneedle, needle.\n\u00e9needle\n", + ); + yield* fileSystem.writeFileString(path.join(cwd, "notes.md"), "needle *.ts\n"); + yield* fileSystem.writeFileString(path.join(cwd, "fold.txt"), "\u017f\ns\n\u212a\nk\n"); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd, "content"); + const result = yield* searchIndex.searchContents({ + query, + limit: 500, + caseSensitive, + wholeWord: true, + useRegex: false, + }); + + expect( + result.matches + .map( + (match) => + `${match.path}:${match.lineNumber}:${match.matchRanges.map((range) => `${range.start}-${range.end}`).join(",")}`, + ) + .toSorted(), + ).toEqual(expected); + }).pipe(Effect.scoped, Effect.provide(testLayer)), +); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 13a54f27e40d6..a5503a50f55b8 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -285,6 +285,28 @@ function buildContentSearchQuery(input: Omit) : { searchQuery: input.query.toLowerCase(), regexMode: false }; } +const ASCII_WORD = /^\w+$/; + +// A whole-word search for an ASCII word asks fff for the word itself, so substring hits are dropped +// natively instead of filling pages with lines the JS check then discards. `-u` keeps `\b` and `i` +// to ASCII: plain search does not fold `ſ` or `K`, and any non-ASCII neighbour stays a candidate. +function buildWholeWordPattern(query: string, caseSensitive: boolean): string { + return `(?${caseSensitive ? "" : "i"}-u)\\b${query}\\b`; +} + +// fff's ranges for that pattern are not used: the JS word check decides, on every hit in the line. +function findWordRanges( + line: string, + word: string, + caseSensitive: boolean, +): Array<{ readonly start: number; readonly end: number }> { + // No `u` flag: with it, `i` would fold `ſ` and `K` onto ASCII letters. + return Array.from(line.matchAll(new RegExp(word, caseSensitive ? "g" : "gi")), (match) => ({ + start: match.index, + end: match.index + match[0].length, + })); +} + function mapContentMatchRanges( line: string, byteRanges: ReadonlyArray, @@ -1318,7 +1340,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( ); contentIndexPending = false; } - let { searchQuery, regexMode } = buildContentSearchQuery(input); + // fff reads glob and path syntax out of a query even in regex mode, so only a bare word qualifies. + const wholeWordPrefilter = input.wholeWord && !input.useRegex && ASCII_WORD.test(input.query); + let { searchQuery, regexMode } = wholeWordPrefilter + ? { searchQuery: buildWholeWordPattern(input.query, input.caseSensitive), regexMode: true } + : buildContentSearchQuery(input); const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing. const rawPageSize = input.wholeWord @@ -1351,9 +1377,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( for (const match of result.items) { if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue; - const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( - (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), - ); + const matchRanges = ( + wholeWordPrefilter + ? findWordRanges(match.lineContent, input.query, input.caseSensitive) + : mapContentMatchRanges(match.lineContent, match.matchRanges) + ).filter((range) => !input.wholeWord || isWholeWordRange(match.lineContent, range)); // fff reports hits past its 512-char lineContent cap with no ranges, so whole words can't be verified there. if (input.wholeWord && matchRanges.length === 0) continue; matches.push({