Add ignoreWhitespace search flag

.changeset/search-ignore-whitespace.md

@@ -0,0 +1,8 @@
+---
+'@embedpdf/engine-core': minor
+'@embedpdf/engine-services': minor
+'@cloudpdf/engine': minor
+'@cloudpdf/server': minor
+---
+
+Adds the `ignoreWhitespace` search flag: a literal query drops whitespace on both sides instead of collapsing it, so `invoice` finds the letter-spaced `i n v o i c e` that OCR'd scans and tracked-out headings produce (and `total amount` finds `totalamount`). Hits span the original text including the dropped whitespace; with `wholeWord` the boundaries are checked on the original text. Like `matchDiacritics`, the flag is literal-only — `regex: true` + `ignoreWhitespace` is rejected with `InvalidArg` (`ignore-whitespace-with-regex`). The flag rides the search token and the cloud search route (`ignoreWhitespace=true`).

cloudpdf/engine/src/document/CloudDocumentSearchService.ts

@@ -126,5 +126,6 @@ function queryIdentity(query: SearchQuery): string {
     query.matchCase ? 1 : 0,
     query.matchDiacritics ? 1 : 0,
     query.wholeWord ? 1 : 0,
+    query.ignoreWhitespace ? 1 : 0,
   ]);
 }

cloudpdf/server/src/routes/search.ts

@@ -205,14 +205,15 @@ function searchStateFromParams(params: unknown): SearchGetState {
     throw new EngineError(EngineErrorCode.InvalidArg, 'search param "q" is required');
   }
   // One flat query shape — flags are independent params; semantic
-  // validation (regex dialect, regex+matchDiacritics) happens in the
-  // engine's validateSearchQuery, not here.
+  // validation (regex dialect, regex+matchDiacritics/ignoreWhitespace)
+  // happens in the engine's validateSearchQuery, not here.
   const query: SearchQuery = {
     text: q,
     ...(bool('regex') ? { regex: true } : {}),
     ...(bool('matchCase') ? { matchCase: true } : {}),
     ...(bool('matchDiacritics') ? { matchDiacritics: true } : {}),
     ...(bool('wholeWord') ? { wholeWord: true } : {}),
+    ...(bool('ignoreWhitespace') ? { ignoreWhitespace: true } : {}),
   };
 
   const maxPages = int('maxPages', 1);

packages/engine/core/src/conformance/runSearchConformance.ts

@@ -270,18 +270,42 @@ export function runSearchConformance(
       }
     });
 
-    test('regex + matchDiacritics is rejected with InvalidArg', async () => {
+    test('ignoreWhitespace keeps every default hit and finds the space-free needle', async () => {
       const doc = await openFixture(engine, opts);
       try {
-        let caught: unknown;
-        try {
-          await doc.search.query({
-            query: { text: fixture.presentRegex, regex: true, matchDiacritics: true },
-          });
-        } catch (err) {
-          caught = err;
+        const key = (m: SearchMatch) => `${m.pageObjectNumber}:${m.charStart}:${m.charCount}`;
+        const plain = await collectAll(doc, { query: { text: fixture.presentLiteral } });
+        // Dropping whitespace can only ADD matches over the collapsing default
+        // fold — every default hit survives, at the same place.
+        const relaxed = await collectAll(doc, {
+          query: { text: fixture.presentLiteral, ignoreWhitespace: true },
+        });
+        const relaxedKeys = new Set(relaxed.matches.map(key));
+        for (const m of plain.matches) expect(relaxedKeys.has(key(m))).toBe(true);
+        // ...and the needle no longer needs the page's spaces.
+        const squashed = await collectAll(doc, {
+          query: { text: fixture.presentLiteral.replace(/\s+/g, ''), ignoreWhitespace: true },
+        });
+        expect(squashed.matches.map(key)).toEqual(relaxed.matches.map(key));
+      } finally {
+        await doc.close();
+      }
+    });
+
+    test('regex + matchDiacritics / ignoreWhitespace are rejected with InvalidArg', async () => {
+      const doc = await openFixture(engine, opts);
+      try {
+        for (const flags of [{ matchDiacritics: true }, { ignoreWhitespace: true }]) {
+          let caught: unknown;
+          try {
+            await doc.search.query({
+              query: { text: fixture.presentRegex, regex: true, ...flags },
+            });
+          } catch (err) {
+            caught = err;
+          }
+          expect(EngineError.is(caught, EngineErrorCode.InvalidArg)).toBe(true);
         }
-        expect(EngineError.is(caught, EngineErrorCode.InvalidArg)).toBe(true);
       } finally {
         await doc.close();
       }

packages/engine/core/src/search/epoch.ts

@@ -45,7 +45,9 @@ function fnv1a64(input: string): string {
  * untouched (their raw form IS the query).
  */
 export function canonicalSearchQuery(query: SearchQuery): SearchQuery {
-  if (query.regex || query.matchCase || query.matchDiacritics) return query;
+  if (query.regex || query.matchCase || query.matchDiacritics || query.ignoreWhitespace) {
+    return query;
+  }
   const canonical: SearchQuery = { text: foldText(query.text).folded };
   if (query.wholeWord) canonical.wholeWord = true;
   return canonical;

packages/engine/core/src/search/fold.ts

@@ -5,7 +5,8 @@
  *
  * Fold version 1:
  *   - whitespace runs collapse to a single space (any `\s`, including the
- *     spaces some compatibility decompositions emit),
+ *     spaces some compatibility decompositions emit) — or are dropped
+ *     entirely with `dropWhitespace` (ignoreWhitespace),
  *   - each code point is NFKD-decomposed (ligatures split: "fi" → "fi",
  *     "²" → "2"),
  *   - combining marks are stripped unless `keepMarks`,
@@ -30,6 +31,8 @@ export interface FoldOptions {
   keepCase?: boolean;
   /** Preserve combining marks (matchDiacritics). */
   keepMarks?: boolean;
+  /** Drop whitespace instead of collapsing it to one space (ignoreWhitespace). */
+  dropWhitespace?: boolean;
 }
 
 export interface FoldedText {
@@ -64,12 +67,12 @@ export function foldText(original: string, options: FoldOptions = {}): FoldedTex
       if (!options.keepCase) piece = piece.toUpperCase().toLowerCase();
     }
     // A decomposition can itself contain whitespace (U+00A8 → space +
-    // combining diaeresis), so collapse runs at the unit level, not just
-    // for source whitespace.
+    // combining diaeresis), so collapse (or drop) runs at the unit level,
+    // not just for source whitespace.
     for (let u = 0; u < piece.length; u++) {
       const unit = piece[u];
       if (WHITESPACE.test(unit)) {
-        if (lastWasSpace) continue;
+        if (options.dropWhitespace || lastWasSpace) continue;
         units.push(' ');
         map.push(index);
         lastWasSpace = true;

packages/engine/core/src/search/literal.ts

@@ -9,7 +9,11 @@ import type { SearchQuery } from './types';
  * queries fold the original page text at query time).
  */
 export function foldOptionsFor(query: SearchQuery): FoldOptions {
-  return { keepCase: !!query.matchCase, keepMarks: !!query.matchDiacritics };
+  return {
+    keepCase: !!query.matchCase,
+    keepMarks: !!query.matchDiacritics,
+    dropWhitespace: !!query.ignoreWhitespace,
+  };
 }
 
 // ONE definition of "word character" for the whole search subsystem —
@@ -35,6 +39,25 @@ export function wordAt(text: string, index: number): boolean {
   return WORD_UNIT.test(String.fromCodePoint(text.codePointAt(index)!));
 }
 
+/**
+ * Whether the folded hit at `at` sits on word boundaries. Checked on the
+ * folded plane, except under `ignoreWhitespace`: dropping whitespace glues
+ * neighbouring words together ("i n v o i c e 42" folds to "invoice42"), so there the
+ * boundaries are read off the ORIGINAL text around the mapped range.
+ */
+function isWholeWordHit(
+  haystack: FoldedText,
+  at: number,
+  needleLength: number,
+  query: SearchQuery,
+): boolean {
+  if (!query.ignoreWhitespace) {
+    return !wordBefore(haystack.folded, at) && !wordAt(haystack.folded, at + needleLength);
+  }
+  const { start, length } = toOriginalRange(haystack, at, needleLength);
+  return !wordBefore(haystack.original, start) && !wordAt(haystack.original, start + length);
+}
+
 /**
  * All non-overlapping literal matches, in original code-unit space.
  * `haystack` must have been folded with `foldOptionsFor(query)`.
@@ -50,10 +73,7 @@ export function matchLiteral(haystack: FoldedText, query: SearchQuery): SearchMa
   while (from <= haystack.folded.length - needle.length) {
     const at = haystack.folded.indexOf(needle, from);
     if (at < 0) break;
-    if (
-      query.wholeWord &&
-      (wordBefore(haystack.folded, at) || wordAt(haystack.folded, at + needle.length))
-    ) {
+    if (query.wholeWord && !isWholeWordHit(haystack, at, needle.length, query)) {
       from = at + 1;
       continue;
     }

packages/engine/core/src/search/regex.ts

@@ -105,7 +105,10 @@ export function validateSearchRegex(pattern: string): SearchRegexValidation {
  * reject with `InvalidArg`. Literal queries are always valid (an empty
  * literal simply finds nothing).
  */
-export type SearchQueryIssue = SearchRegexIssue | 'diacritics-with-regex';
+export type SearchQueryIssue =
+  | SearchRegexIssue
+  | 'diacritics-with-regex'
+  | 'ignore-whitespace-with-regex';
 
 export type SearchQueryValidation =
   | { ok: true }
@@ -121,6 +124,14 @@ export function validateSearchQuery(query: SearchQuery): SearchQueryValidation {
         'Diacritic-sensitive matching is not available for regex patterns (regex runs on the raw text plane).',
     };
   }
+  if (query.ignoreWhitespace) {
+    return {
+      ok: false,
+      issue: 'ignore-whitespace-with-regex',
+      message:
+        'Whitespace-insensitive matching is not available for regex patterns (regex runs on the raw text plane; use \\s* in the pattern).',
+    };
+  }
   return validateSearchRegex(query.text);
 }
 

packages/engine/core/src/search/types.ts

@@ -11,11 +11,13 @@ import type { PdfTextSegment } from '../text/layout';
  * | `matchCase`       | fold case        | `i` flag                       |
  * | `wholeWord`       | boundary check   | pattern is `\b(?:…)\b`-wrapped |
  * | `matchDiacritics` | mark fold        | REJECTED (`InvalidArg`)        |
+ * | `ignoreWhitespace`| whitespace drop  | REJECTED (`InvalidArg`)        |
  *
  * Literal queries match over FOLDED text; regex queries run the portable
- * dialect against the RAW page text — which is why `matchDiacritics`
- * cannot apply to them (diacritic folding is a property of the folded
- * text plane). Validate with `validateSearchQuery` for early UI feedback;
+ * dialect against the RAW page text — which is why `matchDiacritics` and
+ * `ignoreWhitespace` cannot apply to them (diacritic folding and whitespace
+ * dropping are properties of the folded text plane; a pattern spells its
+ * own `\s*`). Validate with `validateSearchQuery` for early UI feedback;
  * engines re-validate and reject with `EngineErrorCode.InvalidArg`.
  */
 export interface SearchQuery {
@@ -38,6 +40,17 @@ export interface SearchQuery {
    * LITERAL ONLY: combined with `regex` the query is rejected.
    */
   matchDiacritics?: boolean;
+  /**
+   * Drop whitespace on both sides instead of collapsing it, so "invoice"
+   * finds the letter-spaced "i n v o i c e" that OCR and tracked-out headings
+   * produce, and "total amount" finds "totalamount". Default false — the default
+   * fold collapses whitespace runs to one space, so a needle still has to
+   * carry a space wherever the page does. A hit spans the original text
+   * including the dropped whitespace; with `wholeWord` the boundaries are
+   * checked on the ORIGINAL text (the folded plane has no word gaps left).
+   * LITERAL ONLY: combined with `regex` the query is rejected.
+   */
+  ignoreWhitespace?: boolean;
 }
 
 /**

packages/engine/core/src/wire/schemas.ts

@@ -544,6 +544,7 @@ export const SearchQuerySchema: z.ZodType<SearchQuery> = z.object({
   matchCase: z.boolean().optional(),
   wholeWord: z.boolean().optional(),
   matchDiacritics: z.boolean().optional(),
+  ignoreWhitespace: z.boolean().optional(),
 });
 
 export const SearchModeSchema = z.enum(['rects', 'full']);

packages/engine/core/src/wire/tokenSchemas.ts

@@ -87,6 +87,7 @@ export const SearchTokenSchema = {
   fields: [
     'epoch',
     'format',
+    'ignoreWhitespace',
     'matchCase',
     'matchDiacritics',
     'maxMatches',

packages/engine/core/src/wire/tokens.ts

@@ -149,6 +149,7 @@ export const encodeSearchToken = (input: SearchToken): string => {
     matchCase: q.matchCase ? true : undefined,
     matchDiacritics: q.matchDiacritics ? true : undefined,
     wholeWord: q.wholeWord ? true : undefined,
+    ignoreWhitespace: q.ignoreWhitespace ? true : undefined,
     startPage: input.startPage,
     skip: input.skip > 0 ? input.skip : undefined,
     maxPages: input.budget?.maxPages,
@@ -172,6 +173,7 @@ export const decodeSearchToken = (raw: string): SearchToken => {
     ...(t.matchCase === 'true' ? { matchCase: true } : {}),
     ...(t.matchDiacritics === 'true' ? { matchDiacritics: true } : {}),
     ...(t.wholeWord === 'true' ? { wholeWord: true } : {}),
+    ...(t.ignoreWhitespace === 'true' ? { ignoreWhitespace: true } : {}),
   };
   const maxPages =
     t.maxPages === undefined ? undefined : decodePositiveInteger(t.maxPages, 'maxPages');

packages/engine/core/test/search/epoch.test.ts

@@ -114,5 +114,7 @@ describe('canonicalSearchQuery', () => {
     expect(canonicalSearchQuery(caseSensitive)).toEqual(caseSensitive);
     const regex = { text: 'C\\d+', regex: true } as const;
     expect(canonicalSearchQuery(regex)).toEqual(regex);
+    const whitespaceInsensitive = { text: 'i n v o i c e', ignoreWhitespace: true } as const;
+    expect(canonicalSearchQuery(whitespaceInsensitive)).toEqual(whitespaceInsensitive);
   });
 });

packages/engine/core/test/search/fold.test.ts

@@ -54,6 +54,17 @@ describe('foldText', () => {
     expect(f.folded).toBe('a b');
   });
 
+  test('dropWhitespace removes whitespace and keeps the map on the letters', () => {
+    const f = foldText('i n\tv\noice', { dropWhitespace: true });
+    expect(f.folded).toBe('invoice');
+    expect(Array.from(f.map)).toEqual([0, 2, 4, 6, 7, 8, 9]);
+  });
+
+  test('dropWhitespace also drops whitespace born from decomposition', () => {
+    const f = foldText('a ¨ b', { dropWhitespace: true });
+    expect(f.folded).toBe('ab');
+  });
+
   test('folds compatibility forms', () => {
     expect(foldText('²').folded).toBe('2'); // superscript two
   });

packages/engine/core/test/search/literal.test.ts

@@ -39,6 +39,58 @@ describe('matchLiteral', () => {
     expect(find('hello\n   world', { text: 'hello world' })).toEqual([{ start: 0, length: 14 }]);
   });
 
+  test('by default a needle must carry a space wherever the page does', () => {
+    expect(find('Ref: i n v o i c e 42', { text: 'invoice' })).toEqual([]);
+    expect(find('Invoice 42', { text: 'i n v o i c e' })).toEqual([]);
+  });
+
+  test('ignoreWhitespace finds letter-spaced text and spans the gaps', () => {
+    // The OCR / tracked-out-heading case: "i n v o i c e" on the page, "invoice" typed.
+    expect(find('Ref: i n v o i c e 42', { text: 'invoice', ignoreWhitespace: true })).toEqual([
+      { start: 5, length: 13 },
+    ]);
+  });
+
+  test('ignoreWhitespace drops whitespace on the needle side too', () => {
+    expect(find('Invoice 42', { text: 'i n v o i c e', ignoreWhitespace: true })).toEqual([
+      { start: 0, length: 7 },
+    ]);
+    expect(find('totalamount', { text: 'total amount', ignoreWhitespace: true })).toEqual([
+      { start: 0, length: 11 },
+    ]);
+  });
+
+  test('ignoreWhitespace matches across line wraps without a space in the needle', () => {
+    expect(find('in\nvoice', { text: 'invoice', ignoreWhitespace: true })).toEqual([
+      { start: 0, length: 8 },
+    ]);
+  });
+
+  test('ignoreWhitespace composes with matchCase', () => {
+    expect(
+      find('I n v o i c e', { text: 'invoice', ignoreWhitespace: true, matchCase: true }),
+    ).toEqual([]);
+    expect(
+      find('I n v o i c e', { text: 'Invoice', ignoreWhitespace: true, matchCase: true }),
+    ).toEqual([{ start: 0, length: 13 }]);
+  });
+
+  test('ignoreWhitespace + wholeWord reads boundaries off the original text', () => {
+    // Dropping whitespace glues "i n v o i c e" to the "42" after it on the
+    // folded plane; the original text still has a gap there, so it is a whole word.
+    expect(
+      find('Ref: i n v o i c e 42', { text: 'invoice', ignoreWhitespace: true, wholeWord: true }),
+    ).toEqual([{ start: 5, length: 13 }]);
+    // ...while a hit glued to letters in the ORIGINAL is still rejected.
+    expect(
+      find('the invoices', { text: 'invoice', ignoreWhitespace: true, wholeWord: true }),
+    ).toEqual([]);
+  });
+
+  test('ignoreWhitespace with a whitespace-only needle finds nothing', () => {
+    expect(find('anything', { text: ' \n ', ignoreWhitespace: true })).toEqual([]);
+  });
+
   test('finds ligature text with a plain-letters needle', () => {
     expect(find('file system', { text: 'file' })).toEqual([{ start: 0, length: 3 }]);
   });

packages/engine/core/test/search/regex.test.ts

@@ -128,6 +128,12 @@ describe('matchRegex', () => {
       /diacritics-with-regex/,
     );
   });
+
+  test('throws on regex + ignoreWhitespace', () => {
+    expect(() => matchRegex('x', { ...q('a'), ignoreWhitespace: true })).toThrow(
+      /ignore-whitespace-with-regex/,
+    );
+  });
 });
 
 describe('validateSearchQuery', () => {
@@ -148,11 +154,16 @@ describe('validateSearchQuery', () => {
     });
   });
 
-  test('regex + matchDiacritics is the one rejected flag combo', () => {
+  test('regex + matchDiacritics / ignoreWhitespace are the rejected flag combos', () => {
     expect(validateSearchQuery({ text: 'a', regex: true, matchDiacritics: true })).toMatchObject({
       ok: false,
       issue: 'diacritics-with-regex',
     });
+    expect(validateSearchQuery({ text: 'a', regex: true, ignoreWhitespace: true })).toMatchObject({
+      ok: false,
+      issue: 'ignore-whitespace-with-regex',
+    });
+    expect(validateSearchQuery({ text: 'i n v o i c e', ignoreWhitespace: true }).ok).toBe(true);
     // every other combination is legal
     expect(
       validateSearchQuery({ text: 'a', regex: true, matchCase: true, wholeWord: true }).ok,