diff --git a/apps/server/src/orchestration-v2/ThreadSearch.test.ts b/apps/server/src/orchestration-v2/ThreadSearch.test.ts index 399224533089d..fa31ee922e134 100644 --- a/apps/server/src/orchestration-v2/ThreadSearch.test.ts +++ b/apps/server/src/orchestration-v2/ThreadSearch.test.ts @@ -24,6 +24,7 @@ import * as Path from "effect/Path"; import * as SqlClient from "effect/sql/SqlClient"; import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations"; import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences"; +import { EXACT_PHRASE_GAP } from "@t3tools/shared/paletteSearch"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import { rebuildMessageSearch } from "./messageSearchIndex.ts"; @@ -291,7 +292,7 @@ it.layer(layerTest)("ThreadSearch", (it) => { for (const match of ['"needle"', null]) { const plan = yield* sql<{ readonly detail: string; - }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`; + }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", hitLimit: 2000, projectId: null, excludedThreadIds: "[]" })}`; assert.isFalse( plan.some((row) => row.detail.includes("TEMP B-TREE")), plan.map((row) => row.detail).join("\n"), @@ -695,6 +696,56 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => { }), ); + it.effect("finds a query in double quotes only where its words are together", () => + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const search = yield* ThreadSearch.ThreadSearch; + const sql = yield* SqlClient.SqlClient; + const project = ProjectId.make("project:exact"); + const together = ThreadId.make("thread:exact-together"); + const wrapped = ThreadId.make("thread:exact-wrapped"); + const apart = ThreadId.make("thread:exact-apart"); + const apartText = `missing notes. ${"Other words. ".repeat(8)}The osprey files`; + // Wrapped as far apart as the matcher accepts: the index must still read it. + const wrappedText = ["still missing", "osprey", "files"].join( + `\n${" ".repeat(EXACT_PHRASE_GAP - 1)}`, + ); + yield* createProject(project); + yield* Effect.forEach( + [ + thread(together, project), + message(together, "exact-together", "user", "the Missing osprey files turned up"), + message(together, "exact-short-word", "user", "an osprey is back"), + thread(wrapped, project), + message(wrapped, "exact-wrapped", "assistant", wrappedText), + thread(apart, project), + message(apart, "exact-apart", "user", apartText), + ], + projections.apply, + { discard: true }, + ); + const threadIds = (query: string) => + search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId))); + assert.sameMembers(yield* threadIds("missing osprey files"), [together, wrapped, apart]); + assert.sameMembers(yield* threadIds('"missing osprey files"'), [together, wrapped]); + // Newer messages with the words apart, or close but not together, do not hide the phrase: + // the first would fill the candidates the index reads, the second the hits it returns. + const addNewer = (id: string, text: string, count: number) => sql` + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${count}) + INSERT INTO orchestration_v2_projection_messages + (message_id, thread_id, role, streaming, created_at, updated_at, payload_json) + SELECT ${id} || i, ${apart}, 'user', 0, + ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', ${text}) + FROM n + `; + yield* addNewer("exact-close:", "missing the osprey files", 2000); + // Nor when the index can read only one of the phrase's words. + assert.sameMembers(yield* threadIds('"osprey is"'), [together]); + yield* addNewer("exact-apart:", apartText, 10_000); + assert.sameMembers(yield* threadIds('"missing osprey files"'), [together, wrapped]); + }), + ); + it.effect("keeps a thread picked for the query when the limit cuts", () => Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; diff --git a/apps/server/src/orchestration-v2/ThreadSearch.ts b/apps/server/src/orchestration-v2/ThreadSearch.ts index 3ad48b26f172d..b0f47af2ee149 100644 --- a/apps/server/src/orchestration-v2/ThreadSearch.ts +++ b/apps/server/src/orchestration-v2/ThreadSearch.ts @@ -24,6 +24,7 @@ import * as SqlClient from "effect/sql/SqlClient"; import * as SqlSchema from "effect/sql/SqlSchema"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import { + EXACT_PHRASE_GAP, messageSearchTokens, foldSearchQuery, rankMessageMatches, @@ -76,6 +77,8 @@ const SearchRequest = Schema.Struct({ ), /** For each short word outside ASCII, the spellings a hit must contain one of. */ shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))), + /** How many of the newest hits to read. */ + hitLimit: Schema.Number, projectId: Schema.NullOr(ProjectId), /** Threads the caller already lists, left out before the hit limit. */ excludedThreadIds: TokenList, @@ -164,13 +167,23 @@ function buildMessageSearch( query: string, hasIndex: boolean, ): Omit | null { - const { long, short } = messageSearchTokens(query); + const { long, short, words, exact } = messageSearchTokens(query); if (long.length === 0) return null; // Two-character CJK words are too short for trigrams. const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3); const narrowing = short.filter((token) => !NON_ASCII.test(token)); + const phrases = long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" "); + // An exact query's words sit together, so the index reads only hits that hold them that + // close. The distance covers the widest phrase the matcher accepts, so it never drops a match. + const near = + exact && long.length > 1 + ? words.join(" ").length + (words.length - 1) * EXACT_PHRASE_GAP + : null; return { - match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null, + match: !indexed ? null : near === null ? phrases : `NEAR(${phrases}, ${near})`, + // The matcher drops most hits that hold a phrase's words but not together, so a + // phrase reads every candidate: the newest ones would otherwise hide an older match. + hitLimit: exact && words.length > 1 ? MESSAGE_CANDIDATE_LIMIT : MESSAGE_HIT_LIMIT, substrings: (indexed ? narrowing : [...long, ...narrowing]).map((token) => ({ token, accentInsensitive: accentInsensitiveGlob(token), @@ -219,7 +232,14 @@ export function buildSearchSnippet(text: string, matchStart: number): string { // accented queries and non-Latin case can still miss: "fermée"/"fermee", "ОШИБКА"/"ошибка". export const messageHitsQuery = ( sql: SqlClient.SqlClient, - { match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded, + { + match, + substrings, + shortSpellings, + hitLimit, + projectId, + excludedThreadIds, + }: typeof SearchRequest.Encoded, ) => sql` WITH eligible AS MATERIALIZED ( SELECT threads.thread_id, threads.project_id @@ -318,7 +338,7 @@ export const messageHitsQuery = ( -- The index's own rowid, so FTS5 returns hits newest first and stops at -- the limit; ordering by the messages table's rowid sorts every hit. ORDER BY "messageRowid" DESC - LIMIT ${MESSAGE_HIT_LIMIT} + LIMIT ${hitLimit} `; const TitleCandidateRow = Schema.Struct({ diff --git a/apps/web/src/components/chat/threadFindStore.test.ts b/apps/web/src/components/chat/threadFindStore.test.ts index 3819d8a4110f7..d49e6c93724bd 100644 --- a/apps/web/src/components/chat/threadFindStore.test.ts +++ b/apps/web/src/components/chat/threadFindStore.test.ts @@ -95,5 +95,8 @@ describe("findForMessageMatch", () => { expect(findForMessageMatch({ snippet: "then Vp run dev again" }, "vp run dev")).toEqual({ query: "Vp run dev", }); + expect(findForMessageMatch({ snippet: "cafe opening; Café open" }, '"cafe open"')).toEqual({ + query: "Café open", + }); }); }); diff --git a/apps/web/src/components/chat/threadFindStore.ts b/apps/web/src/components/chat/threadFindStore.ts index ea52de6c2747e..c4761aa959167 100644 --- a/apps/web/src/components/chat/threadFindStore.ts +++ b/apps/web/src/components/chat/threadFindStore.ts @@ -177,6 +177,10 @@ export function findForMessageMatch( query: string, ): PendingThreadFind { const start = match.messageId ? { start: { entryId: match.messageId, occurrence: 0 } } : {}; + if (messageSearchTokens(query).exact) { + const range = findSearchTokenRanges(match.snippet, query)[0]; + if (range) return { query: match.snippet.slice(range.start, range.end), ...start }; + } // Apostrophes are straight in the query; straightening keeps offsets. const together = messagePhrase(query)?.exec(match.snippet.replace(/[\u2018\u2019]/g, "'")); if (together) { diff --git a/docs/user/thread-sidebar.md b/docs/user/thread-sidebar.md index 905f3a5bfbe9c..1425f38a6db88 100644 --- a/docs/user/thread-sidebar.md +++ b/docs/user/thread-sidebar.md @@ -230,6 +230,8 @@ you last picked for a query ranks higher when you type that query again. Threads whose messages contain all your words appear below, under **In messages**, and open at the matching message. Message search needs at least one word of three or more letters, or two for Chinese, Japanese, and Korean. +Put a search in double quotes, such as `"missing position files"`, to find only +titles and messages that hold those words together, exactly as typed. On mobile, the thread list search also lists message matches and opens them at the matching message; on an iPad with a keyboard, `Cmd+K` opens the same palette. diff --git a/packages/shared/src/paletteSearch.test.ts b/packages/shared/src/paletteSearch.test.ts index 14f7f37dc9205..1460685d28f98 100644 --- a/packages/shared/src/paletteSearch.test.ts +++ b/packages/shared/src/paletteSearch.test.ts @@ -387,9 +387,56 @@ describe("scoreThreadSearch beyond Latin text and titles", () => { ).toContain("VS Code remote support"); }); + it("matches a query in double quotes only as whole words together, as typed", () => { + const threads = [ + thread("Fix sidebar lag"), + thread("Sidebar scroll lag"), + thread("Fix sidebar lagging"), + thread("Lag in the sidebar"), + ]; + expect(rankTitles("sidebar lag", threads)).toHaveLength(4); + expect(rankTitles('"sidebar lag"', threads)).toEqual(["Fix sidebar lag"]); + expect(rankTitles("“Sidebar Lag”", threads)).toEqual(["Fix sidebar lag"]); + // No typos and no optional filler. + expect(rankTitles('"sidebra lag"', threads)).toEqual([]); + expect(rankTitles('"fix the sidebar"', threads)).toEqual([]); + // Without its closing quote the query is still being typed. + expect(rankTitles('"sidebar lag', threads)).toHaveLength(4); + // Whitespace runs in a title do not hide the phrase. + expect(rankTitles('"sidebar lag"', [thread("Fix sidebar lag")])).toEqual(["Fix sidebar lag"]); + // A long run of quotes inside a query folds at once, and quotes around nothing match nothing. + const quotes = '"'.repeat(34); + expect(foldSearchQuery(`a${quotes}x`)).toBe(`a${quotes}x`); + expect(scoreThreadSearch(foldSearchQuery('"#"'), thread('Fix "" handling'), NOW)).toBeNull(); + // A query folded elsewhere may hold stray spaces. + expect(scoreThreadSearch('" lag"', thread("Fix sidebar lag"), NOW)).not.toBeNull(); + const subject = thread("Trim model lists", { + branch: "t3/cleanup-payloads", + pullRequests: [{ number: 22889, title: "Smaller MCP payloads" }], + }); + expect( + scoreThreadSearch(foldSearchQuery('"mcp payloads"'), subject, NOW)?.explain?.(), + ).toMatchObject({ + label: "PR", + text: "#22889 Smaller MCP payloads", + ranges: [{ start: 15, end: 27 }], + }); + expect(scoreThreadSearch(foldSearchQuery('"payloads mcp"'), subject, NOW)).toBeNull(); + // A branch or pull request pasted in quotes still finds its thread. + for (const query of ['"t3/cleanup-payloads"', '"#22889"', '"pr 22889"']) { + expect(scoreThreadSearch(foldSearchQuery(query), subject, NOW), query).not.toBeNull(); + } + expect(scoreSearchTerms(foldSearchQuery('"pull requests"'), ["Open pull requests"])).not.toBe( + null, + ); + expect(scoreSearchTerms(foldSearchQuery('"pull request"'), ["Open pull requests"])).toBeNull(); + }); + it("stores picked queries without splitting an emoji", () => { expect(pickedQuery(`${"a".repeat(31)}🚀`)).toBe("a".repeat(31)); expect(pickedQuery(" Upl ")).toBe("upl"); + // A pick counts for its words, in quotes or not. + expect(pickedQuery('"Fix Sidebar"')).toBe("fix sidebar"); // Folded before it is cut, as stored and as compared. const spaced = "investigate workspace thread reset bug"; expect(pickedQuery(spaced)).toBe(pickedQuery(foldSearchQuery(spaced))); @@ -694,6 +741,15 @@ describe("findSearchTokenRanges", () => { expect(findSearchTokenRanges("drafts, enableFTS", "fts")).toEqual([{ start: 14, end: 17 }]); expect(findSearchTokenRanges("Open handlers.ts", ".ts")).toEqual([{ start: 13, end: 16 }]); }); + + it("highlights a query in double quotes only where its words are together", () => { + expect( + findSearchTokenRanges("files, then the Missing osprey files", '"missing osprey files"'), + ).toEqual([{ start: 16, end: 37 }]); + expect(findSearchTokenRanges("models and the model", '"model"')).toEqual([ + { start: 15, end: 20 }, + ]); + }); }); describe("scoreMessageMatch", () => { @@ -723,6 +779,19 @@ describe("scoreMessageMatch", () => { ["明天部署部署到生产环境", "部署", { quality: 1, offset: 2 }], ["gannet’s gannet's", "gannet's", { quality: 1, offset: 9 }], ["gannet's gannet’s", "gannet’s", { quality: 1, offset: 0 }], + // In double quotes, only the words together as typed, as whole words. + ["the missing osprey files", '"missing osprey files"', { quality: 1.25, offset: 4 }], + ["Missing osprey\n files", "“missing osprey files”", { quality: 1.25, offset: 0 }], + ["missing the osprey files", '"missing osprey files"', null], + ["osprey files missing", '"missing osprey files"', null], + // No further apart than the server's index reads. + [`missing${" ".repeat(17)}osprey files`, '"missing osprey files"', null], + // Whole words by the rule single words follow. + ["quickFix JSONSchema", '"fix JSON"', { quality: 1.25, offset: 5 }], + // Past a place where the phrase only starts a longer word, emoji first or not. + ["🚀 launching, then 🚀 launch", '"🚀 launch"', { quality: 1.25, offset: 19 }], + ["changed file", '"file"', { quality: 1, offset: 8 }], + ["changed files", '"file"', null], ] as const) { expect(scoreMessageMatch(text, query), `${query} in ${text}`).toEqual(expected); } diff --git a/packages/shared/src/paletteSearch.ts b/packages/shared/src/paletteSearch.ts index 02e29a02f2a5d..f95cbd9354f01 100644 --- a/packages/shared/src/paletteSearch.ts +++ b/packages/shared/src/paletteSearch.ts @@ -131,17 +131,49 @@ function prepareSearchField(source: string): SearchField { return field; } -// Quotes around a pasted query: "fix sidebar". -const WRAPPING_QUOTES = /^\s*["'`“”‘’]+|["'`“”‘’]+\s*$/g; +const QUOTE_OR_SPACE = /["'`“”‘’\s]/; +const DOUBLE_QUOTE = /["“”]/; +const WHITESPACE = /\s/; + +// Without the quotes and spaces around a query, so its words never start or end with a +// quote: 'fix sidebar'. A loop: a pattern for mixed runs backtracks on a long run of quotes. +function unwrapQuery(query: string): string { + let start = 0; + let end = query.length; + while (start < end && QUOTE_OR_SPACE.test(query[start]!)) start++; + while (end > start && QUOTE_OR_SPACE.test(query[end - 1]!)) end--; + return query.slice(start, end); +} -/** Folded, whitespace-collapsed query; empty when there is nothing to match. */ -export function foldSearchQuery(query: string): string { - return foldInPlace(query.replace(WRAPPING_QUOTES, "").normalize("NFC")) +/** Double quotes around the whole query ask for its words together, as typed: "fix sidebar". */ +function isExactQuery(query: string): boolean { + const trimmed = query.trim(); + return trimmed.length > 2 && DOUBLE_QUOTE.test(trimmed[0]!) && DOUBLE_QUOTE.test(trimmed.at(-1)!); +} + +function foldQueryWords(query: string): string { + return foldInPlace(unwrapQuery(query).normalize("NFC")) .split(/\s+/) .filter((token) => token.length > 0) .join(" "); } +/** + * Folded, whitespace-collapsed query; empty when there is nothing to match. + * An exact query keeps straight double quotes around its words, so scoring sees it. + */ +export function foldSearchQuery(query: string): string { + const words = foldQueryWords(query); + return words.length > 0 && isExactQuery(query) ? `"${words}"` : words; +} + +/** The words of a folded exact query, or null when the query is not exact. */ +function exactPhrase(normalizedQuery: string): string | null { + return normalizedQuery.length > 2 && normalizedQuery[0] === '"' && normalizedQuery.endsWith('"') + ? normalizedQuery.slice(1, -1) + : null; +} + function wordEnd(field: SearchField, index: number): number { let end = index + 1; while (end < field.text.length && !field.wordStarts[end] && ALPHANUMERIC.test(field.text[end]!)) { @@ -394,6 +426,64 @@ function queryTokens(normalizedQuery: string) { return lastQuery; } +// Where `words` sit side by side in `field` as whole words, whatever whitespace is between them. +function findWholePhrase(field: SearchField, words: ReadonlyArray): SearchRange | null { + const { text } = field; + const first = words[0]!; + // A phrase that opens with punctuation ("#12 fix") brings its own boundary, as a token does. + const ownBoundary = !ALPHANUMERIC.test(first[0]!) && !SYMBOL.test(first[0]!); + for (let start = text.indexOf(first); start !== -1; start = text.indexOf(first, start + 1)) { + if (!field.wordStarts[start] && !ownBoundary) continue; + let end = start + first.length; + for (let word = 1; word < words.length && end !== -1; word++) { + let next = end; + while (next < text.length && WHITESPACE.test(text[next]!)) next++; + end = next > end && text.startsWith(words[word]!, next) ? next + words[word]!.length : -1; + } + if (end === -1) continue; + const endsWord = + end >= text.length || field.wordStarts[end] === 1 || !ALPHANUMERIC.test(text[end]!); + if (endsWord) return { start, end }; + } + return null; +} + +/** + * An exact query: one field must hold its words together as typed, as whole + * words. Typos, joined words, mid-word hits and optional filler do not count. + */ +function matchExactPhrase( + phrase: string, + fields: ReadonlyArray, +): SearchFieldsMatch | null { + // An empty word would match everywhere, and loop forever in `findWholePhrase`. + const words = phrase.split(" ").filter((word) => word.length > 0); + if (words.length === 0) return null; + let best = 0; + let primaryRange: SearchRange | undefined; + for (const [position, weighted] of fields.entries()) { + if (weighted.value.length === 0) continue; + const range = findWholePhrase(preparedField(weighted), words); + if (range === null) continue; + best = Math.max(best, weighted.weight); + if (position === 0) primaryRange = range; + } + if (best === 0) return null; + const wholeName = + primaryRange?.start === 0 && primaryRange.end === preparedField(fields[0]!).text.length; + const bonus = wholeName + ? WHOLE_NAME - WHOLE_WORD + : primaryRange !== undefined && words.length > 1 + ? 0.25 + : 0; + return { + quality: best + bonus, + wholeWords: best >= WHOLE_WORD, + strong: true, + primaryRanges: primaryRange ? [primaryRange] : [], + }; +} + /** * Every query token must match some field. Quality is the mean of each token's * best weighted match, plus a bonus when the first field holds the whole query @@ -405,6 +495,8 @@ function matchSearchFields( options?: { readonly typos?: boolean }, ): SearchFieldsMatch | null { if (normalizedQuery.length === 0) return null; + const phrase = exactPhrase(normalizedQuery); + if (phrase !== null) return matchExactPhrase(phrase, fields); const primary = fields[0]?.value ? preparedField(fields[0]) : undefined; // An empty token would match everywhere, and loop forever in `matchToken`. const { tokens, fillerOptional } = queryTokens(normalizedQuery); @@ -469,9 +561,10 @@ export function messageSearchTokens(query: string): { readonly short: ReadonlyArray; /** Every token, in the order typed. */ readonly words: ReadonlyArray; + /** The query is in double quotes: only its words together, as typed, match. */ + readonly exact: boolean; } { - const tokens = query - .replace(WRAPPING_QUOTES, "") + const tokens = unwrapQuery(query) // The index stores apostrophes straight, as `indexedMessageText` writes them. .replace(CURLY_APOSTROPHES, "'") .normalize("NFC") @@ -486,23 +579,35 @@ export function messageSearchTokens(query: string): { long: tokens.filter(searchable), short: tokens.filter((token) => !searchable(token)), words: tokens, + exact: isExactQuery(query), }; } +/** + * The most whitespace between two words of an exact query in a message: a line + * break and its indentation. The server's index reads only hits that close, so + * the matcher must not accept more. + */ +export const EXACT_PHRASE_GAP = 16; + +// An exact pattern is global and leaves word boundaries to `findExactPhrase`. +function wordsTogether(words: ReadonlyArray, exact: boolean): RegExp | null { + if (words.length < 2) return null; + const escaped = words.map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")); + return exact + ? new RegExp(escaped.join(`\\s{1,${EXACT_PHRASE_GAP}}`), "giu") + : new RegExp(`(? 1 - ? new RegExp( - `(? word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")) - .join("\\s+")}(?![\\p{L}\\p{N}])`, - "iu", - ) - : null; + const { words, exact } = messageSearchTokens(query); + return wordsTogether(words, exact); } // Words written together, as typed, rank above the same words apart, as in @@ -534,6 +639,26 @@ function endsWordAt(text: string, end: number): boolean { return end >= text.length || !ALPHANUMERIC.test(text[end]!) || startsWordAt(text, end); } +// Where an exact phrase sits in `text` at or after `from` as whole words, by the rule single +// words follow: "JSON" is a word in "JSONSchema". `searched` is `text` folded in place. +function findExactPhrase( + text: string, + searched: string, + pattern: RegExp, + from = 0, +): SearchRange | null { + pattern.lastIndex = from; + for (let match = pattern.exec(searched); match !== null; match = pattern.exec(searched)) { + const end = match.index + match[0].length; + if (startsWordAt(text, match.index) && endsWordAt(text, end)) { + return { start: match.index, end }; + } + // A whole character on: from an emoji's second half the pattern finds this match again. + pattern.lastIndex = match.index + (searched.codePointAt(match.index)! > 0xffff ? 2 : 1); + } + return null; +} + // A commit SHA or UUID part: seven or more hex characters with a digit. const HEX_ID = /^(?=[a-f]*\d)[0-9a-f]{7,}$/i; @@ -597,7 +722,7 @@ export interface MessageMatch { * query is prepared once, since the server grades thousands of hits for it. */ export function messageMatcher(query: string): (text: string) => MessageMatch | null { - const { long, short, words } = messageSearchTokens(query); + const { long, short, words, exact } = messageSearchTokens(query); const tokens = [...long, ...short].map((token) => ({ token, // Marks go from the folded token: lowercasing "İ" adds one. @@ -611,8 +736,9 @@ export function messageMatcher(query: string): (text: string) => MessageMatch | const phrase = messagePhrase(query); // Accents typed or left out still read as written together: "cafe open" // in "Café open". - const foldedPhrase = messagePhrase( - words.map((word) => foldInPlace(word).replace(COMBINING_MARKS, "")).join(" "), + const foldedPhrase = wordsTogether( + words.map((word) => foldInPlace(word).replace(COMBINING_MARKS, "")), + exact, ); const queryNonAscii = NON_ASCII.test(query); return (text) => { @@ -641,12 +767,20 @@ export function messageMatcher(query: string): (text: string) => MessageMatch | // Written together as typed: the snippet starts there. Apostrophes are // straight in the query, and straightening keeps offsets. const straight = (value: string) => value.replace(CURLY_APOSTROPHES, "'"); + const phraseAt = (pattern: RegExp | null, searched: string) => + pattern === null + ? null + : exact + ? (findExactPhrase(text, searched, pattern)?.start ?? null) + : (pattern.exec(searched)?.index ?? null); const together = - phrase?.exec(straight(plain)) ?? + phraseAt(phrase, straight(plain)) ?? (queryNonAscii || NON_ASCII.test(text) - ? foldedPhrase?.exec(straight((folded ??= foldInPlace(text)))) + ? phraseAt(foldedPhrase, straight((folded ??= foldInPlace(text)))) : null); - if (together) return { quality: quality + MESSAGE_PHRASE_BONUS, offset: together.index }; + if (together !== null) return { quality: quality + MESSAGE_PHRASE_BONUS, offset: together }; + // An exact query matches only written together; a single word must be whole. + if (exact && (words.length > 1 || quality < 1)) return null; // Else the shortest stretch that holds every word: the snippet starts // there, so it shows them all, and words close together rank above words // far apart ("lakebed perf"). @@ -697,11 +831,31 @@ function shortestSpan(places: ReadonlyArray>): { * Snippets vary per query, so they skip the field cache. */ export function findSearchTokenRanges(text: string, query: string): SearchRange[] { + const { long, words, exact } = messageSearchTokens(query); + // An exact query lights up only where its words are written together. + const phrase = exact + ? wordsTogether( + words.map((word) => foldInPlace(word)), + true, + ) + : null; + if (phrase) { + const searched = foldInPlace(text); + const ranges: SearchRange[] = []; + for ( + let found = findExactPhrase(text, searched, phrase); + found !== null; + found = findExactPhrase(text, searched, phrase, found.end) + ) { + ranges.push(found); + } + return ranges; + } const tokens = new Map( - messageSearchTokens(query) - .long.map( + long + .map( (token) => - [foldSearchQuery(token), Array.from(token).length < MESSAGE_WORD_START_LENGTH] as const, + [foldQueryWords(token), Array.from(token).length < MESSAGE_WORD_START_LENGTH] as const, ) // Some tokens fold to nothing ("´´´"), which would match everywhere. .filter(([token]) => token.length > 0), @@ -713,7 +867,11 @@ export function findSearchTokenRanges(text: string, query: string): SearchRange[ // A token that opens with punctuation brings its own boundary, as in scoring. const ownBoundary = !ALPHANUMERIC.test(token[0]!) && !SYMBOL.test(token[0]!); for (let index = field.text.indexOf(token); index !== -1;) { - if (needsWordStart && !ownBoundary && !field.wordStarts[index]) { + // An exact single word lights up only as a whole word, as it matches. + const skipped = exact + ? !startsWordAt(text, index) || !endsWordAt(text, index + token.length) + : needsWordStart && !ownBoundary && !field.wordStarts[index]; + if (skipped) { index = field.text.indexOf(token, index + 1); continue; } @@ -768,7 +926,8 @@ const THREAD_SEARCH_PICK_MAX_LENGTH = 32; * it, so a long pick matches its query again. */ export function pickedQuery(query: string): string { - const cut = foldSearchQuery(query).slice(0, THREAD_SEARCH_PICK_MAX_LENGTH); + // Without an exact query's quotes: the cut could keep one and drop the other. + const cut = foldQueryWords(query).slice(0, THREAD_SEARCH_PICK_MAX_LENGTH); return (/[\ud800-\udbff]$/.test(cut) ? cut.slice(0, -1) : cut).trim(); } @@ -1010,13 +1169,15 @@ let lastFieldQuery: { readonly query: string; readonly fieldQuery: string } | un // Strip pull request prefixes and the worktree prefix so queries match the stored fields. function threadFieldQuery(normalizedQuery: string): string { if (lastFieldQuery?.query !== normalizedQuery) { + const phrase = exactPhrase(normalizedQuery); + const stripped = (phrase ?? normalizedQuery) + .replace(/^(?:pr|pull request|pull)\s*(?=#?\d)/, "") + .replace(/^#\s*/, "") + .replace(WORKTREE_BRANCH_PREFIX_PATTERN, "") + .replace(FLAT_BRANCH_QUERY_PREFIX_PATTERN, "$1"); lastFieldQuery = { query: normalizedQuery, - fieldQuery: normalizedQuery - .replace(/^(?:pr|pull request|pull)\s*(?=#?\d)/, "") - .replace(/^#\s*/, "") - .replace(WORKTREE_BRANCH_PREFIX_PATTERN, "") - .replace(FLAT_BRANCH_QUERY_PREFIX_PATTERN, "$1"), + fieldQuery: phrase === null || stripped.length === 0 ? stripped : `"${stripped}"`, }; } return lastFieldQuery.fieldQuery; @@ -1043,7 +1204,9 @@ export function scoreThreadSearch( const fields = threadSearchFields(thread); const fieldQuery = threadFieldQuery(normalizedQuery); const fieldMatch = matchSearchFields(fieldQuery, fields, options); - const referenceMatch = fieldMatch ? null : matchReference(normalizedQuery, thread); + const referenceMatch = fieldMatch + ? null + : matchReference(exactPhrase(normalizedQuery) ?? normalizedQuery, thread); const match = fieldMatch ?? referenceMatch; if (match === null) return null; // A row whose title shows no match must say why. With several words, one @@ -1251,12 +1414,13 @@ export function scoreSearchTerms( terms: ReadonlyArray, options?: { readonly typos?: boolean }, ): SearchTermsMatch | null { + const words = exactPhrase(normalizedQuery) ?? normalizedQuery; // Filler words ("the ci") do not make a short query long. - const meaningful = normalizedQuery + const meaningful = words .split(" ") .filter((token) => !FILLER_WORDS.has(token)) .join(" "); - const short = (meaningful || normalizedQuery).length < KEYWORD_QUERY_MIN_LENGTH; + const short = (meaningful || words).length < KEYWORD_QUERY_MIN_LENGTH; const match = matchSearchFields( normalizedQuery, terms.map((value, index) =>