From f082bf3e4a44e4336bf1efda9777f1a1a0f7c5ad Mon Sep 17 00:00:00 2001 From: Aviv Dolev Date: Tue, 8 Sep 2026 11:19:14 +0300 Subject: [PATCH] Add ignoreWhitespace search flag A literal query with ignoreWhitespace drops whitespace on both sides of the fold instead of collapsing it, so "invoice" finds the letter-spaced "i n v o i c e" that OCR'd scans and tracked-out headings produce, and "total amount" finds "totalamount". Hits span the original text including the dropped whitespace; with wholeWord the boundaries are read off the original text, since the folded plane has no word gaps left. Like matchDiacritics the flag is literal-only: regex + ignoreWhitespace is rejected with InvalidArg (ignore-whitespace-with-regex). The flag is part of the cursor/query identity, rides the search token, and is accepted by the cloud search route as ignoreWhitespace=true. Adds letter_spaced_text.pdf (authored by a committed deterministic generator) and a local-engine test that exercises the flag against real page text: letter-spaced hits on both pages, a word split across a line break, a glued word found by a spaced needle, wholeWord boundaries from the original text, and matchCase composition. --- .changeset/search-ignore-whitespace.md | 8 ++ .../document/CloudDocumentSearchService.ts | 1 + cloudpdf/server/src/routes/search.ts | 5 +- .../src/conformance/runSearchConformance.ts | 42 ++++-- packages/engine/core/src/search/epoch.ts | 4 +- packages/engine/core/src/search/fold.ts | 11 +- packages/engine/core/src/search/literal.ts | 30 ++++- packages/engine/core/src/search/regex.ts | 13 +- packages/engine/core/src/search/types.ts | 19 ++- packages/engine/core/src/wire/schemas.ts | 1 + packages/engine/core/src/wire/tokenSchemas.ts | 1 + packages/engine/core/src/wire/tokens.ts | 2 + .../engine/core/test/search/epoch.test.ts | 2 + packages/engine/core/test/search/fold.test.ts | 11 ++ .../engine/core/test/search/literal.test.ts | 52 ++++++++ .../engine/core/test/search/regex.test.ts | 13 +- .../engine/core/test/wire/searchToken.test.ts | 9 ++ packages/engine/main/test/fixtures/README.md | 16 +++ .../generate-letter-spaced-fixture.mjs | 111 ++++++++++++++++ .../main/test/fixtures/letter_spaced_text.pdf | Bin 0 -> 2208 bytes .../test/search-ignore-whitespace.test.ts | 120 ++++++++++++++++++ .../src/features/search/SearchReader.ts | 12 +- .../features/search/internal/searchCursor.ts | 1 + 23 files changed, 454 insertions(+), 30 deletions(-) create mode 100644 .changeset/search-ignore-whitespace.md create mode 100644 packages/engine/main/test/fixtures/generate-letter-spaced-fixture.mjs create mode 100644 packages/engine/main/test/fixtures/letter_spaced_text.pdf create mode 100644 packages/engine/main/test/search-ignore-whitespace.test.ts diff --git a/.changeset/search-ignore-whitespace.md b/.changeset/search-ignore-whitespace.md new file mode 100644 index 000000000..9f000d4f1 --- /dev/null +++ b/.changeset/search-ignore-whitespace.md @@ -0,0 +1,8 @@ +--- +'@embedpdf/engine-core': minor +'@embedpdf/engine-services': minor +'@cloudpdf/engine': minor +'@cloudpdf/server': minor +--- + +Adds the `ignoreWhitespace` search flag: a literal query drops whitespace on both sides instead of collapsing it, so `invoice` finds the letter-spaced `i n v o i c e` that OCR'd scans and tracked-out headings produce (and `total amount` finds `totalamount`). Hits span the original text including the dropped whitespace; with `wholeWord` the boundaries are checked on the original text. Like `matchDiacritics`, the flag is literal-only — `regex: true` + `ignoreWhitespace` is rejected with `InvalidArg` (`ignore-whitespace-with-regex`). The flag rides the search token and the cloud search route (`ignoreWhitespace=true`). diff --git a/cloudpdf/engine/src/document/CloudDocumentSearchService.ts b/cloudpdf/engine/src/document/CloudDocumentSearchService.ts index e39f5ebfc..a8c64d43b 100644 --- a/cloudpdf/engine/src/document/CloudDocumentSearchService.ts +++ b/cloudpdf/engine/src/document/CloudDocumentSearchService.ts @@ -126,5 +126,6 @@ function queryIdentity(query: SearchQuery): string { query.matchCase ? 1 : 0, query.matchDiacritics ? 1 : 0, query.wholeWord ? 1 : 0, + query.ignoreWhitespace ? 1 : 0, ]); } diff --git a/cloudpdf/server/src/routes/search.ts b/cloudpdf/server/src/routes/search.ts index c9781ef31..7ef7d3072 100644 --- a/cloudpdf/server/src/routes/search.ts +++ b/cloudpdf/server/src/routes/search.ts @@ -205,14 +205,15 @@ function searchStateFromParams(params: unknown): SearchGetState { throw new EngineError(EngineErrorCode.InvalidArg, 'search param "q" is required'); } // One flat query shape — flags are independent params; semantic - // validation (regex dialect, regex+matchDiacritics) happens in the - // engine's validateSearchQuery, not here. + // validation (regex dialect, regex+matchDiacritics/ignoreWhitespace) + // happens in the engine's validateSearchQuery, not here. const query: SearchQuery = { text: q, ...(bool('regex') ? { regex: true } : {}), ...(bool('matchCase') ? { matchCase: true } : {}), ...(bool('matchDiacritics') ? { matchDiacritics: true } : {}), ...(bool('wholeWord') ? { wholeWord: true } : {}), + ...(bool('ignoreWhitespace') ? { ignoreWhitespace: true } : {}), }; const maxPages = int('maxPages', 1); diff --git a/packages/engine/core/src/conformance/runSearchConformance.ts b/packages/engine/core/src/conformance/runSearchConformance.ts index e986d8863..1c4e37391 100644 --- a/packages/engine/core/src/conformance/runSearchConformance.ts +++ b/packages/engine/core/src/conformance/runSearchConformance.ts @@ -270,18 +270,42 @@ export function runSearchConformance( } }); - test('regex + matchDiacritics is rejected with InvalidArg', async () => { + test('ignoreWhitespace keeps every default hit and finds the space-free needle', async () => { const doc = await openFixture(engine, opts); try { - let caught: unknown; - try { - await doc.search.query({ - query: { text: fixture.presentRegex, regex: true, matchDiacritics: true }, - }); - } catch (err) { - caught = err; + const key = (m: SearchMatch) => `${m.pageObjectNumber}:${m.charStart}:${m.charCount}`; + const plain = await collectAll(doc, { query: { text: fixture.presentLiteral } }); + // Dropping whitespace can only ADD matches over the collapsing default + // fold — every default hit survives, at the same place. + const relaxed = await collectAll(doc, { + query: { text: fixture.presentLiteral, ignoreWhitespace: true }, + }); + const relaxedKeys = new Set(relaxed.matches.map(key)); + for (const m of plain.matches) expect(relaxedKeys.has(key(m))).toBe(true); + // ...and the needle no longer needs the page's spaces. + const squashed = await collectAll(doc, { + query: { text: fixture.presentLiteral.replace(/\s+/g, ''), ignoreWhitespace: true }, + }); + expect(squashed.matches.map(key)).toEqual(relaxed.matches.map(key)); + } finally { + await doc.close(); + } + }); + + test('regex + matchDiacritics / ignoreWhitespace are rejected with InvalidArg', async () => { + const doc = await openFixture(engine, opts); + try { + for (const flags of [{ matchDiacritics: true }, { ignoreWhitespace: true }]) { + let caught: unknown; + try { + await doc.search.query({ + query: { text: fixture.presentRegex, regex: true, ...flags }, + }); + } catch (err) { + caught = err; + } + expect(EngineError.is(caught, EngineErrorCode.InvalidArg)).toBe(true); } - expect(EngineError.is(caught, EngineErrorCode.InvalidArg)).toBe(true); } finally { await doc.close(); } diff --git a/packages/engine/core/src/search/epoch.ts b/packages/engine/core/src/search/epoch.ts index 62461c013..de705456b 100644 --- a/packages/engine/core/src/search/epoch.ts +++ b/packages/engine/core/src/search/epoch.ts @@ -45,7 +45,9 @@ function fnv1a64(input: string): string { * untouched (their raw form IS the query). */ export function canonicalSearchQuery(query: SearchQuery): SearchQuery { - if (query.regex || query.matchCase || query.matchDiacritics) return query; + if (query.regex || query.matchCase || query.matchDiacritics || query.ignoreWhitespace) { + return query; + } const canonical: SearchQuery = { text: foldText(query.text).folded }; if (query.wholeWord) canonical.wholeWord = true; return canonical; diff --git a/packages/engine/core/src/search/fold.ts b/packages/engine/core/src/search/fold.ts index 455ec83e4..5852e4e00 100644 --- a/packages/engine/core/src/search/fold.ts +++ b/packages/engine/core/src/search/fold.ts @@ -5,7 +5,8 @@ * * Fold version 1: * - whitespace runs collapse to a single space (any `\s`, including the - * spaces some compatibility decompositions emit), + * spaces some compatibility decompositions emit) — or are dropped + * entirely with `dropWhitespace` (ignoreWhitespace), * - each code point is NFKD-decomposed (ligatures split: "fi" → "fi", * "²" → "2"), * - combining marks are stripped unless `keepMarks`, @@ -30,6 +31,8 @@ export interface FoldOptions { keepCase?: boolean; /** Preserve combining marks (matchDiacritics). */ keepMarks?: boolean; + /** Drop whitespace instead of collapsing it to one space (ignoreWhitespace). */ + dropWhitespace?: boolean; } export interface FoldedText { @@ -64,12 +67,12 @@ export function foldText(original: string, options: FoldOptions = {}): FoldedTex if (!options.keepCase) piece = piece.toUpperCase().toLowerCase(); } // A decomposition can itself contain whitespace (U+00A8 → space + - // combining diaeresis), so collapse runs at the unit level, not just - // for source whitespace. + // combining diaeresis), so collapse (or drop) runs at the unit level, + // not just for source whitespace. for (let u = 0; u < piece.length; u++) { const unit = piece[u]; if (WHITESPACE.test(unit)) { - if (lastWasSpace) continue; + if (options.dropWhitespace || lastWasSpace) continue; units.push(' '); map.push(index); lastWasSpace = true; diff --git a/packages/engine/core/src/search/literal.ts b/packages/engine/core/src/search/literal.ts index b9aaf4d31..627f3df75 100644 --- a/packages/engine/core/src/search/literal.ts +++ b/packages/engine/core/src/search/literal.ts @@ -9,7 +9,11 @@ import type { SearchQuery } from './types'; * queries fold the original page text at query time). */ export function foldOptionsFor(query: SearchQuery): FoldOptions { - return { keepCase: !!query.matchCase, keepMarks: !!query.matchDiacritics }; + return { + keepCase: !!query.matchCase, + keepMarks: !!query.matchDiacritics, + dropWhitespace: !!query.ignoreWhitespace, + }; } // ONE definition of "word character" for the whole search subsystem — @@ -35,6 +39,25 @@ export function wordAt(text: string, index: number): boolean { return WORD_UNIT.test(String.fromCodePoint(text.codePointAt(index)!)); } +/** + * Whether the folded hit at `at` sits on word boundaries. Checked on the + * folded plane, except under `ignoreWhitespace`: dropping whitespace glues + * neighbouring words together ("i n v o i c e 42" folds to "invoice42"), so there the + * boundaries are read off the ORIGINAL text around the mapped range. + */ +function isWholeWordHit( + haystack: FoldedText, + at: number, + needleLength: number, + query: SearchQuery, +): boolean { + if (!query.ignoreWhitespace) { + return !wordBefore(haystack.folded, at) && !wordAt(haystack.folded, at + needleLength); + } + const { start, length } = toOriginalRange(haystack, at, needleLength); + return !wordBefore(haystack.original, start) && !wordAt(haystack.original, start + length); +} + /** * All non-overlapping literal matches, in original code-unit space. * `haystack` must have been folded with `foldOptionsFor(query)`. @@ -50,10 +73,7 @@ export function matchLiteral(haystack: FoldedText, query: SearchQuery): SearchMa while (from <= haystack.folded.length - needle.length) { const at = haystack.folded.indexOf(needle, from); if (at < 0) break; - if ( - query.wholeWord && - (wordBefore(haystack.folded, at) || wordAt(haystack.folded, at + needle.length)) - ) { + if (query.wholeWord && !isWholeWordHit(haystack, at, needle.length, query)) { from = at + 1; continue; } diff --git a/packages/engine/core/src/search/regex.ts b/packages/engine/core/src/search/regex.ts index 9f7621bf1..35f022ee0 100644 --- a/packages/engine/core/src/search/regex.ts +++ b/packages/engine/core/src/search/regex.ts @@ -105,7 +105,10 @@ export function validateSearchRegex(pattern: string): SearchRegexValidation { * reject with `InvalidArg`. Literal queries are always valid (an empty * literal simply finds nothing). */ -export type SearchQueryIssue = SearchRegexIssue | 'diacritics-with-regex'; +export type SearchQueryIssue = + | SearchRegexIssue + | 'diacritics-with-regex' + | 'ignore-whitespace-with-regex'; export type SearchQueryValidation = | { ok: true } @@ -121,6 +124,14 @@ export function validateSearchQuery(query: SearchQuery): SearchQueryValidation { 'Diacritic-sensitive matching is not available for regex patterns (regex runs on the raw text plane).', }; } + if (query.ignoreWhitespace) { + return { + ok: false, + issue: 'ignore-whitespace-with-regex', + message: + 'Whitespace-insensitive matching is not available for regex patterns (regex runs on the raw text plane; use \\s* in the pattern).', + }; + } return validateSearchRegex(query.text); } diff --git a/packages/engine/core/src/search/types.ts b/packages/engine/core/src/search/types.ts index 45f459031..3f3855d63 100644 --- a/packages/engine/core/src/search/types.ts +++ b/packages/engine/core/src/search/types.ts @@ -11,11 +11,13 @@ import type { PdfTextSegment } from '../text/layout'; * | `matchCase` | fold case | `i` flag | * | `wholeWord` | boundary check | pattern is `\b(?:…)\b`-wrapped | * | `matchDiacritics` | mark fold | REJECTED (`InvalidArg`) | + * | `ignoreWhitespace`| whitespace drop | REJECTED (`InvalidArg`) | * * Literal queries match over FOLDED text; regex queries run the portable - * dialect against the RAW page text — which is why `matchDiacritics` - * cannot apply to them (diacritic folding is a property of the folded - * text plane). Validate with `validateSearchQuery` for early UI feedback; + * dialect against the RAW page text — which is why `matchDiacritics` and + * `ignoreWhitespace` cannot apply to them (diacritic folding and whitespace + * dropping are properties of the folded text plane; a pattern spells its + * own `\s*`). Validate with `validateSearchQuery` for early UI feedback; * engines re-validate and reject with `EngineErrorCode.InvalidArg`. */ export interface SearchQuery { @@ -38,6 +40,17 @@ export interface SearchQuery { * LITERAL ONLY: combined with `regex` the query is rejected. */ matchDiacritics?: boolean; + /** + * Drop whitespace on both sides instead of collapsing it, so "invoice" + * finds the letter-spaced "i n v o i c e" that OCR and tracked-out headings + * produce, and "total amount" finds "totalamount". Default false — the default + * fold collapses whitespace runs to one space, so a needle still has to + * carry a space wherever the page does. A hit spans the original text + * including the dropped whitespace; with `wholeWord` the boundaries are + * checked on the ORIGINAL text (the folded plane has no word gaps left). + * LITERAL ONLY: combined with `regex` the query is rejected. + */ + ignoreWhitespace?: boolean; } /** diff --git a/packages/engine/core/src/wire/schemas.ts b/packages/engine/core/src/wire/schemas.ts index d34566833..bd2cdec19 100644 --- a/packages/engine/core/src/wire/schemas.ts +++ b/packages/engine/core/src/wire/schemas.ts @@ -544,6 +544,7 @@ export const SearchQuerySchema: z.ZodType = z.object({ matchCase: z.boolean().optional(), wholeWord: z.boolean().optional(), matchDiacritics: z.boolean().optional(), + ignoreWhitespace: z.boolean().optional(), }); export const SearchModeSchema = z.enum(['rects', 'full']); diff --git a/packages/engine/core/src/wire/tokenSchemas.ts b/packages/engine/core/src/wire/tokenSchemas.ts index db789732b..2486e6392 100644 --- a/packages/engine/core/src/wire/tokenSchemas.ts +++ b/packages/engine/core/src/wire/tokenSchemas.ts @@ -87,6 +87,7 @@ export const SearchTokenSchema = { fields: [ 'epoch', 'format', + 'ignoreWhitespace', 'matchCase', 'matchDiacritics', 'maxMatches', diff --git a/packages/engine/core/src/wire/tokens.ts b/packages/engine/core/src/wire/tokens.ts index 8c26ab655..6083e527e 100644 --- a/packages/engine/core/src/wire/tokens.ts +++ b/packages/engine/core/src/wire/tokens.ts @@ -149,6 +149,7 @@ export const encodeSearchToken = (input: SearchToken): string => { matchCase: q.matchCase ? true : undefined, matchDiacritics: q.matchDiacritics ? true : undefined, wholeWord: q.wholeWord ? true : undefined, + ignoreWhitespace: q.ignoreWhitespace ? true : undefined, startPage: input.startPage, skip: input.skip > 0 ? input.skip : undefined, maxPages: input.budget?.maxPages, @@ -172,6 +173,7 @@ export const decodeSearchToken = (raw: string): SearchToken => { ...(t.matchCase === 'true' ? { matchCase: true } : {}), ...(t.matchDiacritics === 'true' ? { matchDiacritics: true } : {}), ...(t.wholeWord === 'true' ? { wholeWord: true } : {}), + ...(t.ignoreWhitespace === 'true' ? { ignoreWhitespace: true } : {}), }; const maxPages = t.maxPages === undefined ? undefined : decodePositiveInteger(t.maxPages, 'maxPages'); diff --git a/packages/engine/core/test/search/epoch.test.ts b/packages/engine/core/test/search/epoch.test.ts index f4e844d63..44d2b1f64 100644 --- a/packages/engine/core/test/search/epoch.test.ts +++ b/packages/engine/core/test/search/epoch.test.ts @@ -114,5 +114,7 @@ describe('canonicalSearchQuery', () => { expect(canonicalSearchQuery(caseSensitive)).toEqual(caseSensitive); const regex = { text: 'C\\d+', regex: true } as const; expect(canonicalSearchQuery(regex)).toEqual(regex); + const whitespaceInsensitive = { text: 'i n v o i c e', ignoreWhitespace: true } as const; + expect(canonicalSearchQuery(whitespaceInsensitive)).toEqual(whitespaceInsensitive); }); }); diff --git a/packages/engine/core/test/search/fold.test.ts b/packages/engine/core/test/search/fold.test.ts index 9d50dff49..bb257746d 100644 --- a/packages/engine/core/test/search/fold.test.ts +++ b/packages/engine/core/test/search/fold.test.ts @@ -54,6 +54,17 @@ describe('foldText', () => { expect(f.folded).toBe('a b'); }); + test('dropWhitespace removes whitespace and keeps the map on the letters', () => { + const f = foldText('i n\tv\noice', { dropWhitespace: true }); + expect(f.folded).toBe('invoice'); + expect(Array.from(f.map)).toEqual([0, 2, 4, 6, 7, 8, 9]); + }); + + test('dropWhitespace also drops whitespace born from decomposition', () => { + const f = foldText('a ¨ b', { dropWhitespace: true }); + expect(f.folded).toBe('ab'); + }); + test('folds compatibility forms', () => { expect(foldText('²').folded).toBe('2'); // superscript two }); diff --git a/packages/engine/core/test/search/literal.test.ts b/packages/engine/core/test/search/literal.test.ts index 6560bd102..d0fdd3eaf 100644 --- a/packages/engine/core/test/search/literal.test.ts +++ b/packages/engine/core/test/search/literal.test.ts @@ -39,6 +39,58 @@ describe('matchLiteral', () => { expect(find('hello\n world', { text: 'hello world' })).toEqual([{ start: 0, length: 14 }]); }); + test('by default a needle must carry a space wherever the page does', () => { + expect(find('Ref: i n v o i c e 42', { text: 'invoice' })).toEqual([]); + expect(find('Invoice 42', { text: 'i n v o i c e' })).toEqual([]); + }); + + test('ignoreWhitespace finds letter-spaced text and spans the gaps', () => { + // The OCR / tracked-out-heading case: "i n v o i c e" on the page, "invoice" typed. + expect(find('Ref: i n v o i c e 42', { text: 'invoice', ignoreWhitespace: true })).toEqual([ + { start: 5, length: 13 }, + ]); + }); + + test('ignoreWhitespace drops whitespace on the needle side too', () => { + expect(find('Invoice 42', { text: 'i n v o i c e', ignoreWhitespace: true })).toEqual([ + { start: 0, length: 7 }, + ]); + expect(find('totalamount', { text: 'total amount', ignoreWhitespace: true })).toEqual([ + { start: 0, length: 11 }, + ]); + }); + + test('ignoreWhitespace matches across line wraps without a space in the needle', () => { + expect(find('in\nvoice', { text: 'invoice', ignoreWhitespace: true })).toEqual([ + { start: 0, length: 8 }, + ]); + }); + + test('ignoreWhitespace composes with matchCase', () => { + expect( + find('I n v o i c e', { text: 'invoice', ignoreWhitespace: true, matchCase: true }), + ).toEqual([]); + expect( + find('I n v o i c e', { text: 'Invoice', ignoreWhitespace: true, matchCase: true }), + ).toEqual([{ start: 0, length: 13 }]); + }); + + test('ignoreWhitespace + wholeWord reads boundaries off the original text', () => { + // Dropping whitespace glues "i n v o i c e" to the "42" after it on the + // folded plane; the original text still has a gap there, so it is a whole word. + expect( + find('Ref: i n v o i c e 42', { text: 'invoice', ignoreWhitespace: true, wholeWord: true }), + ).toEqual([{ start: 5, length: 13 }]); + // ...while a hit glued to letters in the ORIGINAL is still rejected. + expect( + find('the invoices', { text: 'invoice', ignoreWhitespace: true, wholeWord: true }), + ).toEqual([]); + }); + + test('ignoreWhitespace with a whitespace-only needle finds nothing', () => { + expect(find('anything', { text: ' \n ', ignoreWhitespace: true })).toEqual([]); + }); + test('finds ligature text with a plain-letters needle', () => { expect(find('file system', { text: 'file' })).toEqual([{ start: 0, length: 3 }]); }); diff --git a/packages/engine/core/test/search/regex.test.ts b/packages/engine/core/test/search/regex.test.ts index 8db3cc5aa..c65894bb7 100644 --- a/packages/engine/core/test/search/regex.test.ts +++ b/packages/engine/core/test/search/regex.test.ts @@ -128,6 +128,12 @@ describe('matchRegex', () => { /diacritics-with-regex/, ); }); + + test('throws on regex + ignoreWhitespace', () => { + expect(() => matchRegex('x', { ...q('a'), ignoreWhitespace: true })).toThrow( + /ignore-whitespace-with-regex/, + ); + }); }); describe('validateSearchQuery', () => { @@ -148,11 +154,16 @@ describe('validateSearchQuery', () => { }); }); - test('regex + matchDiacritics is the one rejected flag combo', () => { + test('regex + matchDiacritics / ignoreWhitespace are the rejected flag combos', () => { expect(validateSearchQuery({ text: 'a', regex: true, matchDiacritics: true })).toMatchObject({ ok: false, issue: 'diacritics-with-regex', }); + expect(validateSearchQuery({ text: 'a', regex: true, ignoreWhitespace: true })).toMatchObject({ + ok: false, + issue: 'ignore-whitespace-with-regex', + }); + expect(validateSearchQuery({ text: 'i n v o i c e', ignoreWhitespace: true }).ok).toBe(true); // every other combination is legal expect( validateSearchQuery({ text: 'a', regex: true, matchCase: true, wholeWord: true }).ok, diff --git a/packages/engine/core/test/wire/searchToken.test.ts b/packages/engine/core/test/wire/searchToken.test.ts index 339f2bcc2..4d3fb4fa5 100644 --- a/packages/engine/core/test/wire/searchToken.test.ts +++ b/packages/engine/core/test/wire/searchToken.test.ts @@ -51,6 +51,15 @@ describe('search token codec', () => { expect(decodeSearchToken(encodeSearchToken(token))).toEqual(token); }); + test('round-trips whitespace-insensitive queries', () => { + const token: SearchToken = { + epoch: '00000000000000ff', + query: { text: 'invoice', ignoreWhitespace: true, wholeWord: true }, + skip: 0, + }; + expect(decodeSearchToken(encodeSearchToken(token))).toEqual(token); + }); + test('canonical: defaults are omitted, equal searches are byte-equal', () => { const a = encodeSearchToken({ epoch: 'e', diff --git a/packages/engine/main/test/fixtures/README.md b/packages/engine/main/test/fixtures/README.md index 007dfbc80..332c86f94 100644 --- a/packages/engine/main/test/fixtures/README.md +++ b/packages/engine/main/test/fixtures/README.md @@ -65,3 +65,19 @@ JavaScript → GoTo → Hide `/Next` chain, and one malformed GoTo (no `/D`). Both are single-page with a correct `/Count` so the cloud suite needs no byte patching. Edit the generator, re-run it, and commit both the script and the regenerated PDFs together. + +## Generated search fixture + +`letter_spaced_text.pdf` backs `search-ignore-whitespace.test.ts` and is +authored the same way (deterministic, byte-stable): + +```bash +node packages/engine/main/test/fixtures/generate-letter-spaced-fixture.mjs +``` + +Two pages of standard Helvetica (no embedded fonts) carrying the text shapes +the `ignoreWhitespace` search flag exists for: a letter-spaced +`i n v o i c e` on each page, a tracked-out `INVOICE` heading (character +spacing), a mixed-case `I n v o i c e`, a plain `Invoice 42`, the glued +`totalamount` next to the spaced `total amount`, `in` / `voice` split across +a line break, and `the invoices` as the whole-word trap. diff --git a/packages/engine/main/test/fixtures/generate-letter-spaced-fixture.mjs b/packages/engine/main/test/fixtures/generate-letter-spaced-fixture.mjs new file mode 100644 index 000000000..f4041f20d --- /dev/null +++ b/packages/engine/main/test/fixtures/generate-letter-spaced-fixture.mjs @@ -0,0 +1,111 @@ +// Deterministic generator for fixtures/letter_spaced_text.pdf (byte-stable: no dates, no +// randomness). Exercises the ignoreWhitespace search flag: letter-spaced words, a tracked-out +// heading (character spacing), a glued "totalamount", a line-wrapped "in / voice", and a +// mixed-case "I n v o i c e" for matchCase. Re-run after editing and commit both files: +// node packages/engine/main/test/fixtures/generate-letter-spaced-fixture.mjs +import { writeFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const outputPath = process.argv[2] ?? resolve(here, 'letter_spaced_text.pdf'); + +const PAGE_WIDTH = 595; +const PAGE_HEIGHT = 842; + +function escapePdfText(text) { + return text.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)'); +} + +/** One text line: { text, size?, x?, charSpacing?, bold? } */ +function buildContentStream(lines) { + const operations = []; + let y = PAGE_HEIGHT - 80; + for (const line of lines) { + const size = line.size ?? 12; + const font = line.bold ? '/F2' : '/F1'; + const x = line.x ?? 60; + const charSpacing = line.charSpacing ?? 0; + y -= line.gapBefore ?? 0; + operations.push( + 'BT', + `${font} ${size} Tf`, + `${charSpacing} Tc`, + `1 0 0 1 ${x} ${y} Tm`, + `(${escapePdfText(line.text)}) Tj`, + 'ET', + ); + y -= size * 1.8; + } + return operations.join('\n'); +} + +const PAGE_1 = [ + { text: 'INVOICE', size: 22, bold: true, charSpacing: 6 }, + { text: 'Whitespace-insensitive search sample', size: 10, gapBefore: 6 }, + { text: 'Ref: i n v o i c e 42', gapBefore: 18 }, + { text: 'Letter-spaced heading: I N V O I C E', bold: true }, + { text: 'Mixed case for matchCase: I n v o i c e' }, + { text: 'Plain for comparison: Invoice 42' }, + { text: 'Glued words: totalamount due 1,250.00', gapBefore: 18 }, + { text: 'Spaced words: total amount due 1,250.00' }, + { text: 'Wrapped across lines, first half ends in', gapBefore: 18 }, + { text: 'voice and continues here.' }, + { text: 'Whole-word trap: the invoices were sent.', gapBefore: 18 }, +]; + +const PAGE_2 = [ + { text: 'Page two', size: 16, bold: true }, + { text: 'Second occurrence: i n v o i c e 43', gapBefore: 18 }, + { text: 'Tabular gaps: total amount 99.00' }, + { text: 'Nothing to find here: shipping and handling.', gapBefore: 18 }, +]; + +const objects = []; +function addObject(body) { + objects.push(body); + return objects.length; // 1-based object number +} + +const catalogNumber = 1; +const pagesNumber = 2; +objects.push(null, null); // placeholders for catalog and pages + +const helveticaNumber = addObject( + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>', +); +const helveticaBoldNumber = addObject( + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding >>', +); + +const pageNumbers = [PAGE_1, PAGE_2].map((lines) => { + const content = buildContentStream(lines); + const contentNumber = addObject( + `<< /Length ${Buffer.byteLength(content, 'latin1')} >>\nstream\n${content}\nendstream`, + ); + return addObject( + `<< /Type /Page /Parent ${pagesNumber} 0 R /MediaBox [0 0 ${PAGE_WIDTH} ${PAGE_HEIGHT}] ` + + `/Resources << /Font << /F1 ${helveticaNumber} 0 R /F2 ${helveticaBoldNumber} 0 R >> >> ` + + `/Contents ${contentNumber} 0 R >>`, + ); +}); + +objects[catalogNumber - 1] = `<< /Type /Catalog /Pages ${pagesNumber} 0 R >>`; +objects[pagesNumber - 1] = + `<< /Type /Pages /Kids [${pageNumbers.map((n) => `${n} 0 R`).join(' ')}] /Count ${pageNumbers.length} >>`; + +let pdf = '%PDF-1.4\n%\xE2\xE3\xCF\xD3\n'; +const offsets = []; +objects.forEach((body, index) => { + offsets.push(Buffer.byteLength(pdf, 'latin1')); + pdf += `${index + 1} 0 obj\n${body}\nendobj\n`; +}); +const xrefOffset = Buffer.byteLength(pdf, 'latin1'); +pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; +for (const offset of offsets) pdf += `${String(offset).padStart(10, '0')} 00000 n \n`; +pdf += `trailer\n<< /Size ${objects.length + 1} /Root ${catalogNumber} 0 R >>\nstartxref\n${xrefOffset}\n%%EOF\n`; + +writeFileSync(outputPath, Buffer.from(pdf, 'latin1')); +console.log( + `wrote ${outputPath} (${Buffer.byteLength(pdf, 'latin1')} bytes, ${pageNumbers.length} pages)`, +); diff --git a/packages/engine/main/test/fixtures/letter_spaced_text.pdf b/packages/engine/main/test/fixtures/letter_spaced_text.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a69159b1975175449b7f68e7584594c73f7bb3ae GIT binary patch literal 2208 zcmcIm-A>yu6u!?>oExM~T9Y`5(*~hQgTaQzpP+&Zvh4>xw+GSqd&?m@V3sv%3eg3d zasvq}T*CYJ0hg6ibkC~>onG`sR0eKyuRe0;uZX4Q(jvsOdAebKq);Wgn%`OP-UUG* z+GmD$|9s}nJ-4D{5FN@=RYGEHH$r}pMl1!-4zhoDFxXd3wVfB*^K!!F)YeeY*yU-g z=4=-1kAvtafh2+PB*1)vRS&7$p&1-YZdV)Z;a1I;3GA zxr6@H%rW=}r7N=wR^fuNnO}`+&Vjs3XdK4zR-sdxg=yb`5BR-LXAdL`hdmEn zcW*RnTXexnt&D+2NN(Q2L}+87X3YeU9t;S%RT5c|u5`F_(bbtr>@reicOVNcWjPMj znv5ifz9vhOO}#Y1kzds#>835oavb3`y-|Xddy?V4iH2+cWpO2ba42vel(P5=Tx4l1 R)7IxmiX1;kb`H;vg1_aaKdk@& literal 0 HcmV?d00001 diff --git a/packages/engine/main/test/search-ignore-whitespace.test.ts b/packages/engine/main/test/search-ignore-whitespace.test.ts new file mode 100644 index 000000000..6f098211f --- /dev/null +++ b/packages/engine/main/test/search-ignore-whitespace.test.ts @@ -0,0 +1,120 @@ +import { readFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, test } from 'vitest'; +import type { Engine, SearchMatch, SearchQuery } from '@embedpdf/engine-core/runtime'; +import { createLocalEngine } from '../src/index'; + +const here = dirname(fileURLToPath(import.meta.url)); +// Authored in-repo by fixtures/generate-letter-spaced-fixture.mjs (see fixtures/README.md). +// Two pages: letter-spaced "i n v o i c e" (twice, once per page), a tracked-out INVOICE +// heading, a mixed-case "I n v o i c e", a plain "Invoice 42", glued "totalamount" next to +// spaced "total amount", "in" / "voice" split across a line break, and "the invoices". +const fixturePath = resolve(here, 'fixtures', 'letter_spaced_text.pdf'); + +/** The matched text exactly as the page carries it (snippet, 'full' mode). */ +function hitText(m: SearchMatch): string { + const s = m.snippet!; + return s.text.slice(s.matchStart, s.matchStart + s.matchLength); +} + +/** Case-folded, whitespace-free form — the identity the flag matches on. */ +function squash(text: string): string { + return text.replace(/\s+/g, '').toLowerCase(); +} + +const key = (m: SearchMatch) => `${m.pageObjectNumber}:${m.charStart}:${m.charCount}`; + +describe('ignoreWhitespace against real page text (engine-local, wasm runtime)', () => { + let engine: Engine; + let bytes: Uint8Array; + + beforeAll(async () => { + engine = await createLocalEngine({ runtime: { prefer: 'wasm' } }); + bytes = new Uint8Array(await readFile(fixturePath)); + }); + + afterAll(async () => { + if (engine) await engine.destroy(); + }); + + async function search(query: SearchQuery): Promise { + const doc = await engine.open({ kind: 'bytes', id: 'letter-spaced', bytes }); + try { + const matches: SearchMatch[] = []; + let cursor: string | undefined; + for (;;) { + const slice = await doc.search.query({ query, cursor }); + matches.push(...slice.matches); + if (slice.nextCursor === null) return matches; + cursor = slice.nextCursor; + } + } finally { + await doc.close(); + } + } + + test('the default fold does not see letter-spaced text', async () => { + const hits = await search({ text: 'invoice' }); + expect(hits.length).toBeGreaterThan(0); + for (const m of hits) expect(hitText(m)).not.toMatch(/\s/); + }); + + test('ignoreWhitespace finds the letter-spaced hits on both pages and keeps every default hit', async () => { + const plain = await search({ text: 'invoice' }); + const relaxed = await search({ text: 'invoice', ignoreWhitespace: true }); + + const relaxedKeys = new Set(relaxed.map(key)); + for (const m of plain) expect(relaxedKeys.has(key(m))).toBe(true); + expect(relaxed.length).toBeGreaterThan(plain.length); + + const texts = relaxed.map(hitText); + // The letter-spaced occurrences, one per page — the hit spans the gaps. + expect(texts.filter((t) => t === 'i n v o i c e')).toHaveLength(2); + expect(new Set(relaxed.map((m) => m.pageObjectNumber)).size).toBe(2); + // The word split across a line break. + expect(texts.some((t) => /^in\s+voice$/.test(t))).toBe(true); + // Every hit is the same word once whitespace and case are dropped. + for (const t of texts) expect(squash(t)).toMatch(/^invoices?$/); + // Every hit carries drawable geometry, including the ones spanning gaps. + for (const m of relaxed) expect(m.segments.length).toBeGreaterThan(0); + }); + + test('a spaced needle finds the glued word', async () => { + const plain = (await search({ text: 'total amount' })).map(hitText); + expect(plain).not.toContain('totalamount'); + + const relaxed = (await search({ text: 'total amount', ignoreWhitespace: true })).map(hitText); + expect(relaxed).toContain('totalamount'); + expect(relaxed).toContain('total amount'); + for (const t of relaxed) expect(squash(t)).toBe('totalamount'); + }); + + test('wholeWord boundaries come from the original text', async () => { + const relaxed = await search({ text: 'invoice', ignoreWhitespace: true }); + const whole = await search({ text: 'invoice', ignoreWhitespace: true, wholeWord: true }); + + // "the invoices" is the only hit glued to a word character in the ORIGINAL + // text (the snippet continues with the trailing "s"). + const gluedToWordCharacter = (m: SearchMatch) => { + const s = m.snippet!; + return /[\p{L}\p{N}]/u.test(s.text.charAt(s.matchStart + s.matchLength)); + }; + expect(relaxed.filter(gluedToWordCharacter)).toHaveLength(1); + expect(whole.filter(gluedToWordCharacter)).toHaveLength(0); + expect(whole.length).toBe(relaxed.length - 1); + // "i n v o i c e 42": the digits are glued only on the folded plane. + expect(whole.map(hitText).filter((t) => t === 'i n v o i c e')).toHaveLength(2); + }); + + test('matchCase composes with ignoreWhitespace', async () => { + const lower = await search({ text: 'invoice', ignoreWhitespace: true, matchCase: true }); + expect(lower.length).toBeGreaterThan(0); + for (const m of lower) expect(hitText(m)).toBe(hitText(m).toLowerCase()); + + const capitalised = await search({ text: 'Invoice', ignoreWhitespace: true, matchCase: true }); + expect(capitalised.map(hitText)).toContain('Invoice'); + expect(capitalised.map(hitText)).toContain('I n v o i c e'); + for (const m of capitalised) expect(hitText(m).startsWith('I')).toBe(true); + }); +}); diff --git a/packages/engine/services/src/features/search/SearchReader.ts b/packages/engine/services/src/features/search/SearchReader.ts index 8ff7ac55a..79c735848 100644 --- a/packages/engine/services/src/features/search/SearchReader.ts +++ b/packages/engine/services/src/features/search/SearchReader.ts @@ -64,8 +64,8 @@ export class SearchReader { const mode = request.mode ?? 'full'; // One validator covers everything: regex dialect AND flag combos - // (regex + matchDiacritics is the rejected one). Literal queries are - // always valid. + // (regex + matchDiacritics / ignoreWhitespace are the rejected ones). + // Literal queries are always valid. const valid = validateSearchQuery(query); if (!valid.ok) { throw new EngineError( @@ -128,7 +128,7 @@ export class SearchReader { let ranges: SearchMatchRange[]; if (query.regex) { ranges = matchRegex(text, query); - } else if (query.matchCase || query.matchDiacritics) { + } else if (query.matchCase || query.matchDiacritics || query.ignoreWhitespace) { // Non-default fold options: re-fold the cached raw text per query. ranges = matchLiteral(foldText(text, foldOptionsFor(query)), query); } else { @@ -145,7 +145,11 @@ export class SearchReader { // here — the biased range helper keeps zero-width characters // adjacent to the match OUTSIDE it on both sides. Snippets stay in // text space (their offsets are internal to the snippet string). - const chars = charRangeForTextOffsets(corpus.snapshot, range.start, range.start + range.length); + const chars = charRangeForTextOffsets( + corpus.snapshot, + range.start, + range.start + range.length, + ); matches.push({ pageObjectNumber: pon, charStart: chars.start, diff --git a/packages/engine/services/src/features/search/internal/searchCursor.ts b/packages/engine/services/src/features/search/internal/searchCursor.ts index 1dc989e02..b6065d43b 100644 --- a/packages/engine/services/src/features/search/internal/searchCursor.ts +++ b/packages/engine/services/src/features/search/internal/searchCursor.ts @@ -32,6 +32,7 @@ export function searchQueryKey(query: SearchQuery, mode: string): string { query.matchCase ? 1 : 0, query.matchDiacritics ? 1 : 0, query.wholeWord ? 1 : 0, + query.ignoreWhitespace ? 1 : 0, mode, ]); }