Phone documents and knowledge bases on the shared document engine

App.tsx

@@ -16,7 +16,7 @@
   useProExpiryRedirect,
 } from './src/navigation/useProExpiryRedirect';
 import { useTheme } from './src/theme';
-import { hardwareService, modelManager, authService, ragService, remoteServerManager } from './src/services';
+import { hardwareService, modelManager, authService, knowledgeBase, remoteServerManager } from './src/services';
 import logger from './src/utils/logger';
 import { useAppStore, useAuthStore, useRemoteServerStore, useWhisperStore } from './src/stores';
 import { useDebugLogsStore } from './src/stores/debugLogsStore';
@@ -121,7 +121,7 @@
   );
 }
 
 function App() {
   useEffect(() => () => {
     stopStartupProbe?.();
     stopStartupProbe = null;
@@ -342,7 +342,9 @@
       }
 
       // Initialize RAG database tables
-      ragService.ensureReady().catch((err) => logger.error('Failed to initialize RAG service on startup', err));
+      knowledgeBase.ensureReady().catch((err) => logger.error('Failed to open the knowledge base on startup', err));
+      // The document services register for chat deletions (they free index entries no chat uses).
+      import('./src/services/documents').catch((err) => logger.error('Failed to start document services', err));
 
       try {
         // Register the private Pro entitlement provider before the first status

__tests__/App.test.tsx

@@ -156,7 +156,7 @@ jest.mock('../src/services', () => ({
   authService: {
     hasPassphrase: jest.fn(() => Promise.resolve(false)),
   },
-  ragService: {
+  knowledgeBase: {
     ensureReady: jest.fn(() => Promise.resolve()),
   },
   remoteServerManager: {

__tests__/contracts/ragEmbedding.contract.test.ts

@@ -2,8 +2,8 @@
  * RAG Embedding Contract Tests
  *
  * Documents and verifies the expected interface between our embedding service
- * and the llama.rn native module's embedding API. Also documents the vector
- * storage format and search contract.
+ * and the llama.rn native module's embedding API, and the vector storage format
+ * the document index keeps (services/documents/store.ts).
  *
  * These tests use mocks — they verify interface compatibility and expected
  * data shapes, not actual native functionality.
@@ -116,48 +116,6 @@ describe('RAG Embedding Contract', () => {
     });
   });
 
-  // ============================================================================
-  // Search Result Contract
-  // ============================================================================
-  describe('search result format', () => {
-    it('RagSearchResult uses score instead of rank', () => {
-      const result = {
-        doc_id: 1,
-        name: 'document.pdf',
-        content: 'chunk text',
-        position: 0,
-        score: 0.85,
-      };
-
-      expect(result).toHaveProperty('score');
-      expect(result.score).toBeGreaterThanOrEqual(-1);
-      expect(result.score).toBeLessThanOrEqual(1);
-    });
-
-    it('cosine similarity score range is [-1, 1]', () => {
-      // Identical vectors → 1.0
-      // Orthogonal vectors → 0.0
-      // Opposite vectors → -1.0
-      const scores = [1, 0.85, 0.5, 0, -0.3, -1];
-      scores.forEach(score => {
-        expect(score).toBeGreaterThanOrEqual(-1);
-        expect(score).toBeLessThanOrEqual(1);
-      });
-    });
-
-    it('search results are sorted by descending score', () => {
-      const results = [
-        { score: 0.95 },
-        { score: 0.8 },
-        { score: 0.65 },
-      ];
-
-      for (let i = 1; i < results.length; i++) {
-        expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
-      }
-    });
-  });
-
   // ============================================================================
   // Model Asset Contract
   // ============================================================================
@@ -183,27 +141,4 @@ describe('RAG Embedding Contract', () => {
       expect(destPath).toContain('all-MiniLM-L6-v2-Q8_0.gguf');
     });
   });
-
-  // ============================================================================
-  // IndexProgress Contract
-  // ============================================================================
-  describe('IndexProgress stages', () => {
-    it('includes embedding stage in the pipeline', () => {
-      const stages = ['extracting', 'chunking', 'indexing', 'embedding', 'done'];
-
-      expect(stages).toContain('embedding');
-      expect(stages.indexOf('embedding')).toBe(3);
-      expect(stages.indexOf('done')).toBe(4);
-    });
-
-    it('embedding stage comes after indexing and before done', () => {
-      const stages = ['extracting', 'chunking', 'indexing', 'embedding', 'done'];
-      const embIdx = stages.indexOf('embedding');
-      const idxIdx = stages.indexOf('indexing');
-      const doneIdx = stages.indexOf('done');
-
-      expect(embIdx).toBeGreaterThan(idxIdx);
-      expect(embIdx).toBeLessThan(doneIdx);
-    });
-  });
 });

__tests__/hardening/batch2-retry.test.ts

@@ -27,8 +27,9 @@ jest.mock('../../src/services/hardware', () => ({
   hardwareService: { getAvailableMemoryGB: jest.fn(() => 4), getTotalMemoryGB: jest.fn(() => 8) },
 }));
 jest.mock('../../src/bootstrap/hookRegistry', () => ({
-  HOOKS: { audioStop: 'audio.stop' },
+  HOOKS: { audioStop: 'audio.stop', conversationsDeleted: 'chat.conversationsDeleted' },
   callHook: jest.fn(),
+  registerHook: () => () => undefined,
 }));
 
 import { handleRetryMessageFn } from '../../src/screens/ChatScreen/useChatMessageHandlers';

__tests__/hardening/batch3-documentAttach.test.ts

@@ -2,15 +2,16 @@
  * BATCH 3 — Chat Attachments & Vision (hardening)
  * File 1: Document attachment validation & multi-attachment queueing.
  *
- * Drives the REAL documentService (the attach seam). The only mocked boundary is
- * react-native-fs (a native module) and pdfExtractor (a native module). All
- * validation / extension / size / decode logic runs for real — deleting the
- * implementation would fail these tests.
+ * Drives the REAL documentService (the attach seam). The only stood-in boundaries
+ * are native: react-native-fs (memfs) and op-sqlite (a real in-memory SQLite, so the
+ * background read the attach starts runs for real too). The native PDF reader is
+ * absent, as on a device without it. All validation / extension / size / decode
+ * logic runs for real — deleting the implementation would fail these tests.
  *
  * On-device test-plan cases covered here (see the on-device test plan, Batch 3):
  *  - #2  supported .txt accepted            (COVERED-REAL in existing suite; asserted end-to-end here for the accept-set)
  *  - #12 unsupported binary (.docx) rejected with a visible error
- *  - #13 file > 5MB rejected with a visible error
+ *  - #13 oversized file rejected with a visible error (the limit is 1 GB: documents are read page by page)
  *  - #14 .md accepted
  *  - #15 .json accepted
  *  - #17 URL-encoded filename display-name decode  → BUG-FOUND (service returns it un-decoded)
@@ -28,12 +29,13 @@
   return { __esModule: true, default: boundary.module, ...boundary.module };
 });
 
-jest.mock('../../src/services/pdfExtractor', () => ({
-  pdfExtractor: { isAvailable: jest.fn(() => false), extractText: jest.fn() },
-}));
+jest.mock('@op-engineering/op-sqlite', () => require('../harness/sqliteFake').realSqliteModule());
 
+import { createHash } from 'node:crypto';
 import { documentService } from '../../src/services/documentService';
 
+const GB = 1024 * 1024 * 1024;
+
 /** Put one readable file on the fake device, with independent stored bytes and reported metadata. */
 function seedReadableFile(
   path: string,
@@ -43,6 +45,8 @@
   defaultNativeFileSystemBoundary.seedTextFile(path, content, size);
 }
 
+const sha256 = (content: string) => createHash('sha256').update(content).digest('hex');
+
 describe('Batch3 · document attach validation (real documentService)', () => {
   beforeEach(() => {
     defaultNativeFileSystemBoundary.reset();
@@ -63,14 +67,16 @@
       expect(documentService.isSupported(name)).toBe(true);
     });
 
-    it.each(acceptedNames)('processDocumentFromPath() builds a document attachment for %s', async (name) => {
+    it.each(acceptedNames)('attachDocument() keeps a copy and builds a document attachment for %s', async (name) => {
       seedReadableFile(`/docs/${name}`, 'sample body', 11);
-      const att = await documentService.processDocumentFromPath(`/docs/${name}`, name);
-      expect(att).not.toBeNull();
-      expect(att!.type).toBe('document');
-      expect(att!.fileName).toBe(name);
-      expect(att!.textContent).toBe('sample body');
-      expect(att!.fileSize).toBe(11);
+      const att = await documentService.attachDocument(`/docs/${name}`, name);
+      expect(att.type).toBe('document');
+      expect(att.fileName).toBe(name);
+      expect(att.fileSize).toBe(11);
+      // Its identity in the document index is the content hash; the kept copy holds the same bytes.
+      expect(att.documentId).toBe(sha256('sample body'));
+      expect(att.uri.startsWith(`${defaultNativeFileSystemBoundary.DocumentDirectoryPath}/attachments/`)).toBe(true);
+      expect(await defaultNativeFileSystemBoundary.readAscii(att.uri, 11)).toBe('sample body');
     });
   });
 
@@ -85,112 +91,90 @@
       expect(documentService.isSupported('photo.png')).toBe(false);
     });
 
-    it('processDocumentFromPath() throws an "Unsupported file type" error for .docx (no chip is added)', async () => {
+    it('attachDocument() throws an "Unsupported file type" error for .docx (no chip is added)', async () => {
       await expect(
-        documentService.processDocumentFromPath('/docs/report.docx', 'report.docx'),
+        documentService.attachDocument('/docs/report.docx', 'report.docx'),
       ).rejects.toThrow(/Unsupported file type/);
     });
+
+    it('attachDocument() refuses a PDF on a device without the native PDF reader', async () => {
+      seedReadableFile('/docs/scan.pdf', '%PDF-1.7');
+      await expect(
+        documentService.attachDocument('/docs/scan.pdf', 'scan.pdf'),
+      ).rejects.toThrow(/PDF extraction is not available/);
+    });
   });
 
-  // ── #13: oversized (>5MB) file rejected with a visible error ────────────────
+  // ── #13: oversized file rejected with a visible error ───────────────────────
   describe('oversized files are rejected (#13)', () => {
-    it('rejects a file at 5MB + 1 byte with a "too large" error', async () => {
-      seedReadableFile('/docs/huge.txt', 'x', 5 * 1024 * 1024 + 1);
+    it('rejects a file at 1 GB + 1 byte with a "too large" error', async () => {
+      seedReadableFile('/docs/huge.txt', 'x', GB + 1);
       await expect(
-        documentService.processDocumentFromPath('/docs/huge.txt', 'huge.txt'),
+        documentService.attachDocument('/docs/huge.txt', 'huge.txt'),
       ).rejects.toThrow(/too large/i);
     });
 
-    it('accepts a file exactly at the 5MB boundary', async () => {
-      seedReadableFile('/docs/limit.txt', 'ok', 5 * 1024 * 1024);
-      const att = await documentService.processDocumentFromPath('/docs/limit.txt', 'limit.txt');
-      expect(att).not.toBeNull();
+    it('accepts a file exactly at the 1 GB boundary', async () => {
+      seedReadableFile('/docs/limit.txt', 'ok', GB);
+      const att = await documentService.attachDocument('/docs/limit.txt', 'limit.txt');
+      expect(att.fileSize).toBe(GB);
+    });
+
+    it('says so when the picked file cannot be found', async () => {
+      await expect(
+        documentService.attachDocument('/docs/gone.txt', 'gone.txt'),
+      ).rejects.toThrow(/Could not read this file/);
     });
   });
 
   // ── #17: URL-encoded filename should display decoded ────────────────────────
   //
-  // BUG-FOUND: documentService.processDocumentFromPath returns `fileName` VERBATIM
-  // for display (src/services/documentService.ts:156,190). It decodeURIComponent()s
-  // only the file PATH inside resolveContentUri, never the display name. So a name
-  // like 'my%20notes.txt' is surfaced to the attachment chip un-decoded, contrary
-  // to device case #17 ("shows the human-readable decoded filename, not the raw
-  // encoded string"). On device the picker usually hands back an already-decoded
-  // name, which is why the E2E may still pass — but the service seam does not
-  // guarantee it. Fixing this belongs in src (decode the display name once in the
-  // service); per hardening rules we do NOT edit src, so the correct-behavior
-  // assertion is skipped and the actual (buggy) behavior is pinned below.
+  // BUG-FOUND: documentService.attachDocument returns `fileName` VERBATIM for display. It
+  // decodeURIComponent()s only the file PATH inside resolveContentUri, never the display name. So a name
+  // like 'my%20notes.txt' is surfaced to the attachment chip un-decoded, contrary to device case #17
+  // ("shows the human-readable decoded filename, not the raw encoded string"). On device the picker
+  // usually hands back an already-decoded name, which is why the E2E may still pass — but the service
+  // seam does not guarantee it.
   describe('URL-encoded filename display decode (#17)', () => {
     it.skip('BUG-FOUND: display fileName should be decoded (my%20notes.txt -> "my notes.txt")', async () => {
-      seedReadableFile('/docs/my%20notes.txt', 'body');
-      const att = await documentService.processDocumentFromPath(
-        '/docs/my%20notes.txt',
+      seedReadableFile('/mock/documents/my notes.txt', 'body');
+      const att = await documentService.attachDocument(
+        '/mock/documents/my%20notes.txt',
         'my%20notes.txt',
       );
       // Desired behavior per device case #17: the chip shows the decoded, human-readable name.
-      expect(att!.fileName).toBe('my notes.txt');
-    });
-
-    it.skip('pins ACTUAL behavior: the display fileName is returned un-decoded (documents the bug) — SKIP: do not enshrine the bug as passing; see the desired-behavior skip above', async () => {
-      seedReadableFile('/docs/my%20notes.txt', 'body');
-      const att = await documentService.processDocumentFromPath(
-        '/docs/my%20notes.txt',
-        'my%20notes.txt',
-      );
-      // NOT decoded today — this is the current, incorrect behavior we are pinning.
-      expect(att!.fileName).toBe('my%20notes.txt');
+      expect(att.fileName).toBe('my notes.txt');
     });
 
     it('resolves the file even when the PATH is URL-encoded (path decode works)', async () => {
-      // The path decode DOES happen (resolveContentUri), so a file whose path
-      // carries %20 still reads without error — the attach itself succeeds.
-      seedReadableFile(
-        '/mock/documents/my notes.txt',
-        'decoded path body',
+      // The path decode DOES happen (resolveContentUri), so a file whose path carries %20 is found and
+      // kept — the attach itself succeeds, with the decoded file's bytes.
+      seedReadableFile('/docs/inbox/my notes.txt', 'decoded path body');
+      const att = await documentService.attachDocument(
+        'file:///docs/inbox/my%20notes.txt',
+        'my notes.txt',
       );
-      const att = await documentService.processDocumentFromPath(
-        '/mock/documents/my%20notes.txt',
-        'my%20notes.txt',
-      );
-      expect(att).not.toBeNull();
-      expect(att!.textContent).toBe('decoded path body');
+      expect(att.documentId).toBe(sha256('decoded path body'));
     });
   });
 
   // ── #34/#35: multiple document attachments queue as distinct attachments ────
   describe('multiple document attachments queue (#34, #35)', () => {
-    it('produces two distinct attachments with unique ids for two files', async () => {
+    it('produces two distinct attachments with unique ids and kept copies for two files', async () => {
       seedReadableFile('/docs/a.py', 'py body');
-      const first = await documentService.processDocumentFromPath('/docs/a.py', 'a.py');
-
-      // Advance the clock so the second attachment gets a different id (id is Date.now()).
-      const nowSpy = jest.spyOn(Date, 'now');
-      const base = Date.now();
-      nowSpy.mockReturnValue(base + 5);
       seedReadableFile('/docs/b.ts', 'ts body');
-      const second = await documentService.processDocumentFromPath('/docs/b.ts', 'b.ts');
-      nowSpy.mockRestore();
-
-      expect(first!.fileName).toBe('a.py');
-      expect(second!.fileName).toBe('b.ts');
-      expect(first!.id).not.toBe(second!.id);
-      // Both are documents that would render as side-by-side chips (#34) and send
-      // together in one message (#35).
-      expect(first!.type).toBe('document');
-      expect(second!.type).toBe('document');
-    });
-
-    it('formatForContext() renders each queued document independently for the LLM (#35)', () => {
-      const ctxA = documentService.formatForContext({
-        id: '1', type: 'document', uri: '/x/a.py', fileName: 'a.py', textContent: 'print(1)',
-      });
-      const ctxB = documentService.formatForContext({
-        id: '2', type: 'document', uri: '/x/b.ts', fileName: 'b.ts', textContent: 'const x = 1',
-      });
-      expect(ctxA).toContain('a.py');
-      expect(ctxA).toContain('print(1)');
-      expect(ctxB).toContain('b.ts');
-      expect(ctxB).toContain('const x = 1');
+      const first = await documentService.attachDocument('/docs/a.py', 'a.py');
+      const second = await documentService.attachDocument('/docs/b.ts', 'b.ts');
+
+      expect(first.fileName).toBe('a.py');
+      expect(second.fileName).toBe('b.ts');
+      expect(first.id).not.toBe(second.id);
+      expect(first.uri).not.toBe(second.uri);
+      expect(first.documentId).not.toBe(second.documentId);
+      // Both are documents that would render as side-by-side chips (#34) and send together in one
+      // message (#35); each is read on its own by the document index.
+      expect(first.type).toBe('document');
+      expect(second.type).toBe('document');
     });
   });
 });