From 044d8815601f410bcf6b124b13481cf0e285d790 Mon Sep 17 00:00:00 2001
From: Andre <andre.rabold@gmail.com>
Date: Sat, 22 Aug 2026 09:35:27 -0700
Subject: [PATCH 1/3] fix(store): reclaim sqlite space after documentation
 removal

SQLite DELETE leaves free pages and WAL growth, so the store file can
grow when docs are removed. Checkpoint the WAL after bulk deletes without
blocking readers, and add a compact command that vacuums when idle.

Fixes #476
---
 docs/concepts/data-storage.md               |  9 ++-
 docs/guides/basic-usage.md                  |  3 +-
 skills/docs-manage/SKILL.md                 | 37 +++++++++-
 src/cli/commands/compact.test.ts            | 67 +++++++++++++++++
 src/cli/commands/compact.ts                 | 67 +++++++++++++++++
 src/cli/index.ts                            |  2 +
 src/store/DocumentManagementClient.ts       |  5 ++
 src/store/DocumentManagementService.test.ts | 36 ++++++++++
 src/store/DocumentManagementService.ts      | 42 +++++++++++
 src/store/DocumentStore.test.ts             | 80 +++++++++++++++++++++
 src/store/DocumentStore.ts                  | 80 +++++++++++++++++++++
 src/store/trpc/interfaces.ts                |  8 +++
 src/store/trpc/router.test.ts               |  8 +++
 src/store/trpc/router.ts                    | 18 +++++
 src/store/types.ts                          | 16 +++++
 src/utils/string.test.ts                    | 19 +++++
 src/utils/string.ts                         | 24 +++++++
 17 files changed, 516 insertions(+), 5 deletions(-)
 create mode 100644 src/cli/commands/compact.test.ts
 create mode 100644 src/cli/commands/compact.ts
 create mode 100644 src/utils/string.test.ts

diff --git a/docs/concepts/data-storage.md b/docs/concepts/data-storage.md
index d800c0b2..582d4495 100644
--- a/docs/concepts/data-storage.md
+++ b/docs/concepts/data-storage.md
@@ -419,9 +419,14 @@ Health monitoring capabilities:
 
 ### Maintenance Operations
 
-Regular maintenance tasks:
+SQLite does not shrink the database file when documents are deleted. Deleted rows become free pages, and WAL mode records those deletes in `documents.db-wal`, so the on-disk size can grow even after a remove.
+
+After bulk deletes (`removeVersion` and `removeAllDocuments`) the store runs `wal_checkpoint(PASSIVE)`. That never waits on readers or writers, so search stays available. It truncates WAL frames when nothing else is using the file. It does not shrink `documents.db`.
+
+`VACUUM` takes an exclusive lock and blocks readers. Run `docs-mcp-server compact` (or `compact` against a remote worker with `--server-url`) when the store is idle to rewrite the file and reclaim free pages. In-memory databases skip compaction. Single-page deletes during refresh do not checkpoint or vacuum.
+
+Other regular maintenance:
 
-- VACUUM operations for space recovery
 - Index rebuilding via REINDEX
 - Orphaned record cleanup via foreign key constraints
 - Performance analysis using EXPLAIN QUERY PLAN
diff --git a/docs/guides/basic-usage.md b/docs/guides/basic-usage.md
index b278365e..4e86e1f6 100644
--- a/docs/guides/basic-usage.md
+++ b/docs/guides/basic-usage.md
@@ -81,6 +81,7 @@ npx @arabold/docs-mcp-server@latest fetch-url https://react.dev/reference/react/
 | `find-version <library>` | Resolve the best matching version for a library |
 | `refresh <library>` | Re-scrape an existing library, skipping unchanged pages |
 | `remove <library>` | Delete a library or version from the index |
+| `compact` | Reclaim unused SQLite pages and shrink the store. Takes an exclusive lock and may block searches until it finishes. |
 
 Run `npx @arabold/docs-mcp-server@latest --help` for the full command reference.
 
@@ -88,7 +89,7 @@ Run `npx @arabold/docs-mcp-server@latest --help` for the full command reference.
 
 - Structured commands (`list`, `search`, `find-version`) default to **JSON** on stdout in non-interactive runs.
 - Use `--output json|yaml|toon` to pick a format.
-- Plain-text commands (`fetch-url`, `scrape`, `refresh`, `remove`) write their output directly to stdout.
+- Plain-text commands (`fetch-url`, `scrape`, `refresh`, `remove`, `compact`) write their output directly to stdout.
 - Use `--quiet` to suppress non-error diagnostics or `--verbose` for debug output.
 - In non-interactive runs, diagnostics stay off stdout so agents and scripts can parse results safely.
 
diff --git a/skills/docs-manage/SKILL.md b/skills/docs-manage/SKILL.md
index dd10db33..de018b81 100644
--- a/skills/docs-manage/SKILL.md
+++ b/skills/docs-manage/SKILL.md
@@ -3,8 +3,9 @@ name: docs-manage
 description: >-
   Manage the Grounded Docs MCP Server documentation index. Covers scraping
   and indexing documentation from URLs or local files, refreshing existing
-  indexes with changed content, and removing libraries from the index.
-  Use when you need to add, update, or delete indexed documentation.
+  indexes with changed content, removing libraries from the index, and
+  compacting the SQLite store. Use when you need to add, update, or delete
+  indexed documentation, or reclaim disk space after removals.
 compatibility: Requires Node.js 22+ and npx
 metadata:
   author: grounded.tools
@@ -21,6 +22,7 @@ on stdout.
 - A library is not yet indexed and you need its docs available for search.
 - Documentation may be stale and you want to pull in updated pages.
 - You want to remove a library or version from the index to free space.
+- The store file is still large after removals and you want to compact it.
 
 ## Commands
 
@@ -123,6 +125,34 @@ npx @arabold/docs-mcp-server@latest remove react --version 18.3.1
 ```
 
 This is destructive and cannot be undone. Re-run `scrape` to re-index.
+Bulk deletes checkpoint the WAL without locking readers. Use `compact` when
+idle if the main store file is still large.
+
+### compact
+
+Reclaim unused SQLite pages and truncate the WAL file so the store shrinks
+on disk. This takes an exclusive lock and may block searches until it
+finishes.
+
+```bash
+npx @arabold/docs-mcp-server@latest compact [options]
+```
+
+| Flag | Alias | Description |
+|------|-------|-------------|
+| `--server-url <url>` | | Remote pipeline worker URL |
+| `--quiet` | | Suppress non-error diagnostics |
+| `--verbose` | | Enable debug logging |
+
+Example:
+
+```bash
+npx @arabold/docs-mcp-server@latest compact
+```
+
+Removing documentation does not shrink the main SQLite file. Bulk deletes
+only run a non-blocking WAL checkpoint. Run this command when the store is
+idle to VACUUM and reclaim free pages.
 
 ## Output behaviour
 
@@ -145,6 +175,9 @@ npx @arabold/docs-mcp-server@latest refresh react --version 19.0.0
 
 # 3. Clean up old versions
 npx @arabold/docs-mcp-server@latest remove react --version 18.3.1
+
+# 4. Reclaim unused store space if needed
+npx @arabold/docs-mcp-server@latest compact
 ```
 
 ## Important notes
diff --git a/src/cli/commands/compact.test.ts b/src/cli/commands/compact.test.ts
new file mode 100644
index 00000000..88780eb8
--- /dev/null
+++ b/src/cli/commands/compact.test.ts
@@ -0,0 +1,67 @@
+/** Unit test for compact command */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import yargs from "yargs";
+import { createCompactCommand } from "./compact";
+
+const stdoutWriteMock = vi.fn();
+
+const compactFn = vi.fn(async () => ({
+  skipped: false,
+  vacuumed: true,
+  beforeBytes: 2048,
+  afterBytes: 1024,
+  reclaimedBytes: 1024,
+}));
+vi.mock("../../store", () => ({
+  createDocumentManagement: vi.fn(async () => ({
+    shutdown: vi.fn(),
+    compact: compactFn,
+  })),
+}));
+vi.mock("../utils", () => ({
+  getGlobalOptions: vi.fn(() => ({ storePath: undefined })),
+  getEventBus: vi.fn(() => ({
+    on: vi.fn(),
+    emit: vi.fn(),
+  })),
+  CliContext: {},
+  setupLogging: vi.fn(),
+}));
+vi.mock("../../utils/config", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("../../utils/config")>();
+  return {
+    ...actual,
+    loadConfig: vi.fn(() => ({
+      app: { storePath: "/mock/store" },
+    })),
+  };
+});
+
+describe("compact command", () => {
+  let stdoutWriteSpy: { mockRestore: () => void };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    stdoutWriteMock.mockReset();
+    stdoutWriteSpy = vi
+      .spyOn(process.stdout, "write")
+      .mockImplementation(stdoutWriteMock as any);
+  });
+
+  afterEach(() => {
+    stdoutWriteSpy.mockRestore();
+  });
+
+  it("calls compact with force and reports reclaimed space", async () => {
+    const parser = yargs().scriptName("test");
+    createCompactCommand(parser);
+
+    await parser.parse("compact");
+
+    expect(compactFn).toHaveBeenCalledWith({ force: true });
+    expect(stdoutWriteMock).toHaveBeenCalledWith(
+      "Compacted store from 2.0 KB to 1.0 KB (reclaimed 1.0 KB).\n",
+    );
+  });
+});
diff --git a/src/cli/commands/compact.ts b/src/cli/commands/compact.ts
new file mode 100644
index 00000000..6e8331da
--- /dev/null
+++ b/src/cli/commands/compact.ts
@@ -0,0 +1,67 @@
+/**
+ * Compact command - Reclaims unused SQLite pages and truncates the WAL file.
+ */
+
+import type { Argv } from "yargs";
+import { createDocumentManagement } from "../../store";
+import { TelemetryEvent, telemetry } from "../../telemetry";
+import { loadConfig } from "../../utils/config";
+import { logger } from "../../utils/logger";
+import { formatBytes } from "../../utils/string";
+import { renderTextOutput } from "../output";
+import { type CliContext, getEventBus } from "../utils";
+
+export function createCompactCommand(cli: Argv) {
+  cli.command(
+    "compact",
+    "Reclaim unused space in the document store (exclusive lock; may block searches)",
+    (yargs) => {
+      return yargs.option("server-url", {
+        type: "string",
+        description:
+          "URL of external pipeline worker RPC (e.g., http://localhost:8080/api)",
+        alias: "serverUrl",
+      });
+    },
+    async (argv) => {
+      await telemetry.track(TelemetryEvent.CLI_COMMAND, {
+        command: "compact",
+        useServerUrl: !!argv.serverUrl,
+      });
+
+      const serverUrl = argv.serverUrl as string | undefined;
+      const appConfig = loadConfig(argv, {
+        configPath: argv.config as string,
+        searchDir: argv.storePath as string,
+      });
+
+      const eventBus = getEventBus(argv as CliContext);
+
+      const docService = await createDocumentManagement({
+        serverUrl,
+        eventBus,
+        appConfig,
+      });
+      try {
+        const result = await docService.compact({ force: true });
+
+        if (result.skipped) {
+          renderTextOutput("Skipped compaction for in-memory store.");
+        } else if (result.reclaimedBytes > 0) {
+          renderTextOutput(
+            `Compacted store from ${formatBytes(result.beforeBytes)} to ${formatBytes(result.afterBytes)} (reclaimed ${formatBytes(result.reclaimedBytes)}).`,
+          );
+        } else {
+          renderTextOutput("Store is already compact.");
+        }
+      } catch (error) {
+        logger.error(
+          `❌ Failed to compact store: ${error instanceof Error ? error.message : String(error)}`,
+        );
+        throw error;
+      } finally {
+        await docService.shutdown();
+      }
+    },
+  );
+}
diff --git a/src/cli/index.ts b/src/cli/index.ts
index 550aae08..f66a3d8c 100644
--- a/src/cli/index.ts
+++ b/src/cli/index.ts
@@ -14,6 +14,7 @@ import {
 import { loadConfig } from "../utils/config";
 import { resolveStorePath } from "../utils/paths";
 // Commands
+import { createCompactCommand } from "./commands/compact";
 import { createConfigCommand } from "./commands/config";
 import { createDefaultAction } from "./commands/default";
 import { createFetchUrlCommand } from "./commands/fetchUrl";
@@ -164,6 +165,7 @@ export function createCli(argv: string[]): Argv {
     .showHelpOnFail(true);
 
   // Register Commands
+  createCompactCommand(cli);
   createConfigCommand(cli);
   createDefaultAction(cli);
   createFetchUrlCommand(cli);
diff --git a/src/store/DocumentManagementClient.ts b/src/store/DocumentManagementClient.ts
index d700f474..2ab42384 100644
--- a/src/store/DocumentManagementClient.ts
+++ b/src/store/DocumentManagementClient.ts
@@ -11,6 +11,7 @@ import type { IDocumentManagement } from "./trpc/interfaces";
 import type { DataRouter } from "./trpc/router";
 import type {
   ActivityHistory,
+  CompactResult,
   DbVersionWithLibrary,
   EmbeddingConfigInfo,
   FindVersionResult,
@@ -92,6 +93,10 @@ export class DocumentManagementClient implements IDocumentManagement {
     await this.client.removeAllDocuments.mutate({ library, version: version ?? null });
   }
 
+  async compact(options?: { force?: boolean; vacuum?: boolean }): Promise<CompactResult> {
+    return this.client.compact.mutate(options ?? {});
+  }
+
   async getVersionsByStatus(statuses: VersionStatus[]): Promise<DbVersionWithLibrary[]> {
     return this.client.getVersionsByStatus.query({
       statuses: statuses as unknown as string[],
diff --git a/src/store/DocumentManagementService.test.ts b/src/store/DocumentManagementService.test.ts
index 752004b3..3f778a5d 100644
--- a/src/store/DocumentManagementService.test.ts
+++ b/src/store/DocumentManagementService.test.ts
@@ -39,6 +39,18 @@ const mockStore = {
   queryLibraryVersions: vi.fn().mockResolvedValue(new Map<string, any[]>()),
   addDocuments: vi.fn(),
   deletePages: vi.fn(),
+  compact: vi.fn().mockResolvedValue({
+    skipped: false,
+    vacuumed: true,
+    beforeBytes: 100,
+    afterBytes: 50,
+    reclaimedBytes: 50,
+  }),
+  removeVersion: vi.fn().mockResolvedValue({
+    documentsDeleted: 1,
+    versionDeleted: true,
+    libraryDeleted: false,
+  }),
   // Status tracking methods
   updateVersionStatus: vi.fn(),
   updateVersionProgress: vi.fn(),
@@ -350,6 +362,7 @@ describe("DocumentManagementService", () => {
 
       await docService.removeAllDocuments(library, version);
       expect(mockStore.deletePages).toHaveBeenCalledWith(library, version); // Fix: Use mockStoreInstance
+      expect(mockStore.compact).toHaveBeenCalledWith({ force: false, vacuum: false });
     });
 
     it("should handle removing documents with null/undefined/empty version", async () => {
@@ -362,6 +375,29 @@ describe("DocumentManagementService", () => {
       expect(mockStore.deletePages).toHaveBeenCalledWith(library, ""); // Fix: Use mockStoreInstance
     });
 
+    it("should still remove documents when compaction fails", async () => {
+      mockStore.compact.mockRejectedValueOnce(new Error("checkpoint busy"));
+
+      await expect(
+        docService.removeAllDocuments("test-lib", "1.0.0"),
+      ).resolves.toBeUndefined();
+      expect(mockStore.deletePages).toHaveBeenCalledWith("test-lib", "1.0.0");
+    });
+
+    it("should compact the store", async () => {
+      const compactResult = {
+        skipped: false,
+        vacuumed: true,
+        beforeBytes: 200,
+        afterBytes: 80,
+        reclaimedBytes: 120,
+      };
+      mockStore.compact.mockResolvedValueOnce(compactResult);
+
+      await expect(docService.compact({ force: true })).resolves.toEqual(compactResult);
+      expect(mockStore.compact).toHaveBeenCalledWith({ force: true });
+    });
+
     describe("listVersions", () => {
       it("should return an empty array if the library has no documents", async () => {
         mockStore.queryUniqueVersions.mockResolvedValue([]); // Fix: Use mockStoreInstance
diff --git a/src/store/DocumentManagementService.ts b/src/store/DocumentManagementService.ts
index 683cc5c2..2b39c8c1 100644
--- a/src/store/DocumentManagementService.ts
+++ b/src/store/DocumentManagementService.ts
@@ -10,6 +10,7 @@ import type { Chunk } from "../splitter/types";
 import { telemetry } from "../telemetry";
 import type { AppConfig } from "../utils/config";
 import { logger } from "../utils/logger";
+import { formatBytes } from "../utils/string";
 import { sortVersionsDescending } from "../utils/version";
 import { DocumentRetrieverService } from "./DocumentRetrieverService";
 import { DocumentStore } from "./DocumentStore";
@@ -21,6 +22,7 @@ import {
 } from "./errors";
 import type {
   ActivityHistory,
+  CompactResult,
   DbVersionWithLibrary,
   EmbeddingConfigInfo,
   FindVersionResult,
@@ -377,6 +379,7 @@ export class DocumentManagementService {
     );
     const count = await this.store.deletePages(library, normalizedVersion);
     logger.info(`🗑️ Deleted ${count} documents`);
+    await this.compactAfterDelete();
 
     // Emit library change event
     this.eventBus.emit(EventType.LIBRARY_CHANGE, undefined);
@@ -442,10 +445,49 @@ export class DocumentManagementService {
       }
     }
 
+    await this.compactAfterDelete();
+
     // Emit library change event
     this.eventBus.emit(EventType.LIBRARY_CHANGE, undefined);
   }
 
+  /**
+   * Reclaims unused SQLite pages and truncates the WAL file.
+   * VACUUM takes an exclusive lock; use the CLI/`compact` mutation when idle.
+   *
+   * @param options.force Always VACUUM even if no free pages are detected
+   * @param options.vacuum When `false`, only run a non-blocking WAL checkpoint
+   */
+  async compact(options?: { force?: boolean; vacuum?: boolean }): Promise<CompactResult> {
+    const result = await this.store.compact(options);
+    if (result.skipped) {
+      logger.info("🧹 Skipped compaction for in-memory store");
+    } else if (result.vacuumed) {
+      logger.info(
+        `🧹 Compacted store: ${formatBytes(result.beforeBytes)} → ${formatBytes(result.afterBytes)} (reclaimed ${formatBytes(result.reclaimedBytes)})`,
+      );
+    } else if (options?.vacuum === false) {
+      logger.info("🧹 Checkpointed WAL after delete");
+    } else {
+      logger.info("🧹 No free pages to reclaim");
+    }
+    return result;
+  }
+
+  /**
+   * Best-effort WAL checkpoint after a bulk delete. Does not VACUUM, so readers
+   * stay unblocked. Remove succeeds even if the checkpoint fails.
+   */
+  private async compactAfterDelete(): Promise<void> {
+    try {
+      await this.compact({ force: false, vacuum: false });
+    } catch (error) {
+      logger.error(
+        `❌ Failed to checkpoint store after delete: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
   /**
    * Adds pre-processed content directly to the store.
    * This method is used when content has already been processed by a pipeline,
diff --git a/src/store/DocumentStore.test.ts b/src/store/DocumentStore.test.ts
index 4cdec68c..a9b4f1d2 100644
--- a/src/store/DocumentStore.test.ts
+++ b/src/store/DocumentStore.test.ts
@@ -2597,3 +2597,83 @@ describe("DocumentStore - Embedding Model Change Safety", () => {
     });
   });
 });
+
+describe("DocumentStore - compaction", () => {
+  let store: DocumentStore | undefined;
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(join(tmpdir(), "docs-mcp-compact-"));
+  });
+
+  afterEach(async () => {
+    if (store) {
+      await store.shutdown();
+      store = undefined;
+    }
+    rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  it("skips compaction for in-memory databases", async () => {
+    const cfg = loadConfig();
+    cfg.app.embeddingModel = "";
+    store = new DocumentStore(":memory:", cfg);
+    await store.initialize();
+
+    const result = await store.compact({ force: true });
+    expect(result.skipped).toBe(true);
+    expect(result.vacuumed).toBe(false);
+    expect(result.reclaimedBytes).toBe(0);
+  });
+
+  it("reclaims disk space after deleting documents", async () => {
+    const cfg = loadConfig();
+    cfg.app.embeddingModel = "";
+    store = new DocumentStore(join(tempDir, "documents.db"), cfg);
+    await store.initialize();
+
+    const payload = "x".repeat(50_000);
+    for (let i = 0; i < 20; i++) {
+      await store.addDocuments(
+        "compactlib",
+        "1.0.0",
+        1,
+        createScrapeResult(`Page ${i}`, `https://example.com/page-${i}`, payload),
+      );
+    }
+
+    await store.deletePages("compactlib", "1.0.0");
+
+    const result = await store.compact({ force: false });
+    expect(result.skipped).toBe(false);
+    expect(result.vacuumed).toBe(true);
+    expect(result.afterBytes).toBeLessThan(result.beforeBytes);
+    expect(result.reclaimedBytes).toBeGreaterThan(0);
+
+    const second = await store.compact({ force: false });
+    expect(second.vacuumed).toBe(false);
+  });
+
+  it("does not vacuum when vacuum is false", async () => {
+    const cfg = loadConfig();
+    cfg.app.embeddingModel = "";
+    store = new DocumentStore(join(tempDir, "documents.db"), cfg);
+    await store.initialize();
+
+    const payload = "x".repeat(50_000);
+    for (let i = 0; i < 20; i++) {
+      await store.addDocuments(
+        "compactlib",
+        "1.0.0",
+        1,
+        createScrapeResult(`Page ${i}`, `https://example.com/page-${i}`, payload),
+      );
+    }
+
+    await store.deletePages("compactlib", "1.0.0");
+
+    const result = await store.compact({ force: false, vacuum: false });
+    expect(result.skipped).toBe(false);
+    expect(result.vacuumed).toBe(false);
+  });
+});
diff --git a/src/store/DocumentStore.ts b/src/store/DocumentStore.ts
index 56c275d6..d342a071 100644
--- a/src/store/DocumentStore.ts
+++ b/src/store/DocumentStore.ts
@@ -1,3 +1,4 @@
+import { existsSync, statSync } from "node:fs";
 import type { Embeddings } from "@langchain/core/embeddings";
 import Database, { type Database as DatabaseType } from "better-sqlite3";
 import * as sqliteVec from "sqlite-vec";
@@ -22,6 +23,7 @@ import {
 } from "./errors";
 import type {
   ActivityHistory,
+  CompactResult,
   DbChunkMetadata,
   DbChunkRank,
   ListVersionChunksOptions,
@@ -78,6 +80,7 @@ interface EmbeddingBatchContext {
  */
 export class DocumentStore {
   private readonly config: AppConfig;
+  private readonly dbPath: string;
 
   private readonly db: DatabaseType;
   private embeddings: Embeddings | null = null;
@@ -220,6 +223,7 @@ export class DocumentStore {
     if (!dbPath) {
       throw new StoreError("Missing required database path");
     }
+    this.dbPath = dbPath;
     this.config = appConfig;
     this.dbDimension = this.config.embeddings.vectorDimension;
     this.searchWeightVec = this.config.search.weightVec;
@@ -1039,6 +1043,82 @@ export class DocumentStore {
     this.db.close();
   }
 
+  /**
+   * Combined on-disk size of the database file plus WAL and SHM sidecars.
+   */
+  private getOnDiskSizeBytes(): number {
+    if (this.dbPath === ":memory:") {
+      return 0;
+    }
+
+    let total = 0;
+    for (const filePath of [this.dbPath, `${this.dbPath}-wal`, `${this.dbPath}-shm`]) {
+      if (existsSync(filePath)) {
+        total += statSync(filePath).size;
+      }
+    }
+    return total;
+  }
+
+  /**
+   * Reclaims unused SQLite pages and truncates the WAL file.
+   * In-memory databases are skipped.
+   *
+   * VACUUM takes an exclusive lock and blocks readers, so it runs only when
+   * `vacuum` is not `false` (the explicit compact path). After deletes, pass
+   * `{ vacuum: false }` to checkpoint with PASSIVE, which never waits on
+   * readers or writers.
+   *
+   * @param options.force Always VACUUM even if no free pages are detected
+   * @param options.vacuum When `false`, only run a non-blocking WAL checkpoint
+   * @returns Size before/after compaction and whether VACUUM ran
+   */
+  async compact(options?: { force?: boolean; vacuum?: boolean }): Promise<CompactResult> {
+    if (this.dbPath === ":memory:") {
+      return {
+        skipped: true,
+        vacuumed: false,
+        beforeBytes: 0,
+        afterBytes: 0,
+        reclaimedBytes: 0,
+      };
+    }
+
+    try {
+      const force = options?.force === true;
+      const allowVacuum = options?.vacuum !== false;
+      const beforeBytes = this.getOnDiskSizeBytes();
+
+      // PASSIVE never waits; TRUNCATE is reserved for the exclusive compact path.
+      this.db.pragma(
+        allowVacuum ? "wal_checkpoint(TRUNCATE)" : "wal_checkpoint(PASSIVE)",
+      );
+
+      const freelistCount = Number(
+        this.db.pragma("freelist_count", { simple: true }) ?? 0,
+      );
+      const shouldVacuum = allowVacuum && (force || freelistCount > 0);
+
+      if (shouldVacuum) {
+        this.db.exec("VACUUM");
+        this.db.pragma("journal_mode = WAL");
+        this.db.pragma("wal_autocheckpoint = 1000");
+        this.db.pragma("wal_checkpoint(TRUNCATE)");
+      }
+
+      const afterBytes = this.getOnDiskSizeBytes();
+      return {
+        skipped: false,
+        vacuumed: shouldVacuum,
+        beforeBytes,
+        afterBytes,
+        reclaimedBytes: Math.max(0, beforeBytes - afterBytes),
+      };
+    } catch (error) {
+      throw new ConnectionError("Failed to compact document store", error);
+    }
+  }
+
   /**
    * Creates or reconciles the documents_vec virtual table with configurable dimension.
    * Called after migrations and model change detection. The table is initially created
diff --git a/src/store/trpc/interfaces.ts b/src/store/trpc/interfaces.ts
index 5636572d..d457465c 100644
--- a/src/store/trpc/interfaces.ts
+++ b/src/store/trpc/interfaces.ts
@@ -6,6 +6,7 @@ import type { ScraperOptions } from "../../scraper/types";
 import type { EmbeddingModelConfig } from "../embeddings/EmbeddingConfig";
 import type {
   ActivityHistory,
+  CompactResult,
   DbVersionWithLibrary,
   EmbeddingConfigInfo,
   FindVersionResult,
@@ -39,6 +40,13 @@ export interface IDocumentManagement {
   ): Promise<StoreSearchResult[]>;
   removeAllDocuments(library: string, version?: string | null): Promise<void>;
   removeVersion(library: string, version?: string | null): Promise<void>;
+  /**
+   * Reclaims unused SQLite pages and truncates the WAL file.
+   * VACUUM takes an exclusive lock and may block searches until it finishes.
+   * @param options.force Always VACUUM even if no free pages are detected.
+   * @param options.vacuum When `false`, only run a non-blocking WAL checkpoint.
+   */
+  compact(options?: { force?: boolean; vacuum?: boolean }): Promise<CompactResult>;
 
   // Minimal set used indirectly by pipeline/UI where needed
   getVersionsByStatus(statuses: VersionStatus[]): Promise<DbVersionWithLibrary[]>;
diff --git a/src/store/trpc/router.test.ts b/src/store/trpc/router.test.ts
index 2404a802..923c4477 100644
--- a/src/store/trpc/router.test.ts
+++ b/src/store/trpc/router.test.ts
@@ -158,4 +158,12 @@ describe("dataRouter - chunk explorer procedures", () => {
       });
     });
   });
+
+  describe("compact", () => {
+    it("skips compaction for an in-memory store", async () => {
+      const result = await caller.compact({ force: true });
+      expect(result.skipped).toBe(true);
+      expect(result.vacuumed).toBe(false);
+    });
+  });
 });
diff --git a/src/store/trpc/router.ts b/src/store/trpc/router.ts
index fdb9789b..2141a4a6 100644
--- a/src/store/trpc/router.ts
+++ b/src/store/trpc/router.ts
@@ -133,6 +133,24 @@ export function createDataRouter(trpc: unknown) {
         },
       ),
 
+    compact: tt.procedure
+      .input(
+        z
+          .object({ force: z.boolean().optional(), vacuum: z.boolean().optional() })
+          .optional(),
+      )
+      .mutation(
+        async ({
+          ctx,
+          input,
+        }: {
+          ctx: DataTrpcContext;
+          input: { force?: boolean; vacuum?: boolean } | undefined;
+        }) => {
+          return await ctx.docService.compact(input);
+        },
+      ),
+
     // Status and version helpers
 
     getVersionsByStatus: tt.procedure
diff --git a/src/store/types.ts b/src/store/types.ts
index 3d5a3396..5554d7a1 100644
--- a/src/store/types.ts
+++ b/src/store/types.ts
@@ -423,3 +423,19 @@ export interface VersionComposition {
   /** Pages grouped by MIME type, most common first. */
   mimeTypes: CompositionBucket[];
 }
+
+/**
+ * Result of reclaiming unused SQLite pages and truncating the WAL.
+ */
+export interface CompactResult {
+  /** True when the store is in-memory and cannot reclaim disk space. */
+  skipped: boolean;
+  /** True when VACUUM ran. */
+  vacuumed: boolean;
+  /** Combined size of the database, WAL, and SHM files before compaction. */
+  beforeBytes: number;
+  /** Combined size of the database, WAL, and SHM files after compaction. */
+  afterBytes: number;
+  /** Bytes reclaimed (`beforeBytes - afterBytes`, floored at 0). */
+  reclaimedBytes: number;
+}
diff --git a/src/utils/string.test.ts b/src/utils/string.test.ts
new file mode 100644
index 00000000..8f9f83e0
--- /dev/null
+++ b/src/utils/string.test.ts
@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+import { formatBytes } from "./string";
+
+describe("formatBytes", () => {
+  it("formats zero and invalid values as 0 B", () => {
+    expect(formatBytes(0)).toBe("0 B");
+    expect(formatBytes(-1)).toBe("0 B");
+    expect(formatBytes(Number.NaN)).toBe("0 B");
+  });
+
+  it("formats byte counts under 1 KB without a decimal", () => {
+    expect(formatBytes(512)).toBe("512 B");
+  });
+
+  it("formats larger sizes with one decimal place", () => {
+    expect(formatBytes(1536)).toBe("1.5 KB");
+    expect(formatBytes(1024 * 1024)).toBe("1.0 MB");
+  });
+});
diff --git a/src/utils/string.ts b/src/utils/string.ts
index a110a6be..45968115 100644
--- a/src/utils/string.ts
+++ b/src/utils/string.ts
@@ -5,3 +5,27 @@
 export const fullTrim = (str: string): string => {
   return str.replace(/^[\s\r\n\t]+|[\s\r\n\t]+$/g, "");
 };
+
+/**
+ * Formats a byte count as a human-readable size string using binary units.
+ *
+ * @param bytes Number of bytes
+ * @returns Formatted size such as `12.4 MB`
+ */
+export function formatBytes(bytes: number): string {
+  if (!Number.isFinite(bytes) || bytes <= 0) {
+    return "0 B";
+  }
+  if (bytes < 1024) {
+    return `${Math.round(bytes)} B`;
+  }
+
+  const units = ["KB", "MB", "GB", "TB"] as const;
+  let value = bytes / 1024;
+  let unitIndex = 0;
+  while (value >= 1024 && unitIndex < units.length - 1) {
+    value /= 1024;
+    unitIndex++;
+  }
+  return `${value.toFixed(1)} ${units[unitIndex]}`;
+}

From ee0cd03b02cb42f956337e3ba2a3cdb92e347ea1 Mon Sep 17 00:00:00 2001
From: Andre <andre.rabold@gmail.com>
Date: Sat, 22 Aug 2026 11:30:28 -0700
Subject: [PATCH 2/3] fix(store): handle compaction size races

---
 src/store/DocumentManagementService.ts | 4 ++++
 src/store/DocumentStore.ts             | 8 ++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/store/DocumentManagementService.ts b/src/store/DocumentManagementService.ts
index 2b39c8c1..da714cdb 100644
--- a/src/store/DocumentManagementService.ts
+++ b/src/store/DocumentManagementService.ts
@@ -466,6 +466,10 @@ export class DocumentManagementService {
       logger.info(
         `🧹 Compacted store: ${formatBytes(result.beforeBytes)} → ${formatBytes(result.afterBytes)} (reclaimed ${formatBytes(result.reclaimedBytes)})`,
       );
+    } else if (result.reclaimedBytes > 0) {
+      logger.info(
+        `🧹 Checkpointed store: ${formatBytes(result.beforeBytes)} → ${formatBytes(result.afterBytes)} (reclaimed ${formatBytes(result.reclaimedBytes)})`,
+      );
     } else if (options?.vacuum === false) {
       logger.info("🧹 Checkpointed WAL after delete");
     } else {
diff --git a/src/store/DocumentStore.ts b/src/store/DocumentStore.ts
index d342a071..06fdbdbb 100644
--- a/src/store/DocumentStore.ts
+++ b/src/store/DocumentStore.ts
@@ -1,4 +1,4 @@
-import { existsSync, statSync } from "node:fs";
+import { statSync } from "node:fs";
 import type { Embeddings } from "@langchain/core/embeddings";
 import Database, { type Database as DatabaseType } from "better-sqlite3";
 import * as sqliteVec from "sqlite-vec";
@@ -1053,8 +1053,12 @@ export class DocumentStore {
 
     let total = 0;
     for (const filePath of [this.dbPath, `${this.dbPath}-wal`, `${this.dbPath}-shm`]) {
-      if (existsSync(filePath)) {
+      try {
         total += statSync(filePath).size;
+      } catch (error) {
+        if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) {
+          throw error;
+        }
       }
     }
     return total;

From 6eba99c7f5ea9229428722212cc7638c95e70fb6 Mon Sep 17 00:00:00 2001
From: Andre <andre.rabold@gmail.com>
Date: Sat, 19 Sep 2026 07:00:42 -0700
Subject: [PATCH 3/3] fix(store): reduce vacuum memory pressure

---
 docs/concepts/data-storage.md    |  2 +-
 docs/guides/basic-usage.md       |  2 +-
 skills/docs-manage/SKILL.md      |  7 +++++--
 src/cli/commands/compact.test.ts | 13 +++++++++++--
 src/cli/commands/compact.ts      | 20 +++++++++++++-------
 src/store/DocumentStore.test.ts  | 28 ++++++++++++++++++++++++++++
 src/store/DocumentStore.ts       | 16 +++++++++++++++-
 7 files changed, 74 insertions(+), 14 deletions(-)

diff --git a/docs/concepts/data-storage.md b/docs/concepts/data-storage.md
index 582d4495..9195cc7a 100644
--- a/docs/concepts/data-storage.md
+++ b/docs/concepts/data-storage.md
@@ -423,7 +423,7 @@ SQLite does not shrink the database file when documents are deleted. Deleted row
 
 After bulk deletes (`removeVersion` and `removeAllDocuments`) the store runs `wal_checkpoint(PASSIVE)`. That never waits on readers or writers, so search stays available. It truncates WAL frames when nothing else is using the file. It does not shrink `documents.db`.
 
-`VACUUM` takes an exclusive lock and blocks readers. Run `docs-mcp-server compact` (or `compact` against a remote worker with `--server-url`) when the store is idle to rewrite the file and reclaim free pages. In-memory databases skip compaction. Single-page deletes during refresh do not checkpoint or vacuum.
+`VACUUM` takes an exclusive lock and blocks readers. Run `docs-mcp-server compact` (or `compact` against a remote worker with `--server-url`) when the store is idle to rewrite the file and reclaim free pages. The compact command stores SQLite temporary data on disk instead of in process memory, so it keeps RSS low but needs enough free disk space for SQLite's temporary copy of the database. In-memory databases skip compaction. Single-page deletes during refresh do not checkpoint or vacuum.
 
 Other regular maintenance:
 
diff --git a/docs/guides/basic-usage.md b/docs/guides/basic-usage.md
index 4e86e1f6..1662c47a 100644
--- a/docs/guides/basic-usage.md
+++ b/docs/guides/basic-usage.md
@@ -81,7 +81,7 @@ npx @arabold/docs-mcp-server@latest fetch-url https://react.dev/reference/react/
 | `find-version <library>` | Resolve the best matching version for a library |
 | `refresh <library>` | Re-scrape an existing library, skipping unchanged pages |
 | `remove <library>` | Delete a library or version from the index |
-| `compact` | Reclaim unused SQLite pages and shrink the store. Takes an exclusive lock and may block searches until it finishes. |
+| `compact` | Reclaim unused SQLite pages and shrink the store. Takes an exclusive lock, may block searches, and needs temporary disk space while it runs. |
 
 Run `npx @arabold/docs-mcp-server@latest --help` for the full command reference.
 
diff --git a/skills/docs-manage/SKILL.md b/skills/docs-manage/SKILL.md
index de018b81..5195a9cb 100644
--- a/skills/docs-manage/SKILL.md
+++ b/skills/docs-manage/SKILL.md
@@ -132,7 +132,8 @@ idle if the main store file is still large.
 
 Reclaim unused SQLite pages and truncate the WAL file so the store shrinks
 on disk. This takes an exclusive lock and may block searches until it
-finishes.
+finishes. VACUUM uses temporary files to keep memory usage low, so make sure
+there is enough free disk space for SQLite's temporary copy of the database.
 
 ```bash
 npx @arabold/docs-mcp-server@latest compact [options]
@@ -140,6 +141,7 @@ npx @arabold/docs-mcp-server@latest compact [options]
 
 | Flag | Alias | Description |
 |------|-------|-------------|
+| `--force` | | Run VACUUM even when SQLite reports no free pages |
 | `--server-url <url>` | | Remote pipeline worker URL |
 | `--quiet` | | Suppress non-error diagnostics |
 | `--verbose` | | Enable debug logging |
@@ -152,7 +154,8 @@ npx @arabold/docs-mcp-server@latest compact
 
 Removing documentation does not shrink the main SQLite file. Bulk deletes
 only run a non-blocking WAL checkpoint. Run this command when the store is
-idle to VACUUM and reclaim free pages.
+idle to VACUUM and reclaim free pages. Add `--force` only when you want to
+rewrite the store even if SQLite reports no free pages.
 
 ## Output behaviour
 
diff --git a/src/cli/commands/compact.test.ts b/src/cli/commands/compact.test.ts
index 88780eb8..c5098d8f 100644
--- a/src/cli/commands/compact.test.ts
+++ b/src/cli/commands/compact.test.ts
@@ -53,15 +53,24 @@ describe("compact command", () => {
     stdoutWriteSpy.mockRestore();
   });
 
-  it("calls compact with force and reports reclaimed space", async () => {
+  it("compacts when free pages exist and reports reclaimed space", async () => {
     const parser = yargs().scriptName("test");
     createCompactCommand(parser);
 
     await parser.parse("compact");
 
-    expect(compactFn).toHaveBeenCalledWith({ force: true });
+    expect(compactFn).toHaveBeenCalledWith({ force: false });
     expect(stdoutWriteMock).toHaveBeenCalledWith(
       "Compacted store from 2.0 KB to 1.0 KB (reclaimed 1.0 KB).\n",
     );
   });
+
+  it("forces vacuum when requested", async () => {
+    const parser = yargs().scriptName("test");
+    createCompactCommand(parser);
+
+    await parser.parse("compact --force");
+
+    expect(compactFn).toHaveBeenCalledWith({ force: true });
+  });
 });
diff --git a/src/cli/commands/compact.ts b/src/cli/commands/compact.ts
index 6e8331da..0d84c595 100644
--- a/src/cli/commands/compact.ts
+++ b/src/cli/commands/compact.ts
@@ -16,12 +16,18 @@ export function createCompactCommand(cli: Argv) {
     "compact",
     "Reclaim unused space in the document store (exclusive lock; may block searches)",
     (yargs) => {
-      return yargs.option("server-url", {
-        type: "string",
-        description:
-          "URL of external pipeline worker RPC (e.g., http://localhost:8080/api)",
-        alias: "serverUrl",
-      });
+      return yargs
+        .option("force", {
+          type: "boolean",
+          description: "Run VACUUM even when SQLite reports no free pages",
+          default: false,
+        })
+        .option("server-url", {
+          type: "string",
+          description:
+            "URL of external pipeline worker RPC (e.g., http://localhost:8080/api)",
+          alias: "serverUrl",
+        });
     },
     async (argv) => {
       await telemetry.track(TelemetryEvent.CLI_COMMAND, {
@@ -43,7 +49,7 @@ export function createCompactCommand(cli: Argv) {
         appConfig,
       });
       try {
-        const result = await docService.compact({ force: true });
+        const result = await docService.compact({ force: argv.force === true });
 
         if (result.skipped) {
           renderTextOutput("Skipped compaction for in-memory store.");
diff --git a/src/store/DocumentStore.test.ts b/src/store/DocumentStore.test.ts
index a9b4f1d2..6c719703 100644
--- a/src/store/DocumentStore.test.ts
+++ b/src/store/DocumentStore.test.ts
@@ -2601,6 +2601,10 @@ describe("DocumentStore - Embedding Model Change Safety", () => {
 describe("DocumentStore - compaction", () => {
   let store: DocumentStore | undefined;
   let tempDir: string;
+  type TestDb = {
+    pragma(sql: string, options?: { simple?: boolean }): unknown;
+    exec(sql: string): unknown;
+  };
 
   beforeEach(() => {
     tempDir = mkdtempSync(join(tmpdir(), "docs-mcp-compact-"));
@@ -2676,4 +2680,28 @@ describe("DocumentStore - compaction", () => {
     expect(result.skipped).toBe(false);
     expect(result.vacuumed).toBe(false);
   });
+
+  it("uses file temp storage during vacuum and restores the previous setting", async () => {
+    const cfg = loadConfig();
+    cfg.app.embeddingModel = "";
+    store = new DocumentStore(join(tempDir, "documents.db"), cfg);
+    await store.initialize();
+
+    const db = (store as unknown as { db: TestDb }).db;
+    db.pragma("temp_store = MEMORY");
+
+    const originalExec = db.exec.bind(db);
+    let tempStoreDuringVacuum: number | undefined;
+    db.exec = (sql: string): unknown => {
+      if (sql === "VACUUM") {
+        tempStoreDuringVacuum = Number(db.pragma("temp_store", { simple: true }));
+      }
+      return originalExec(sql);
+    };
+
+    await store.compact({ force: true });
+
+    expect(tempStoreDuringVacuum).toBe(1);
+    expect(Number(db.pragma("temp_store", { simple: true }))).toBe(2);
+  });
 });
diff --git a/src/store/DocumentStore.ts b/src/store/DocumentStore.ts
index 06fdbdbb..ed6fbaed 100644
--- a/src/store/DocumentStore.ts
+++ b/src/store/DocumentStore.ts
@@ -1064,6 +1064,20 @@ export class DocumentStore {
     return total;
   }
 
+  /**
+   * Runs VACUUM with temp storage on disk so large stores do not duplicate the
+   * compacted database in process memory.
+   */
+  private vacuumWithFileTempStore(): void {
+    const prevTempStore = Number(this.db.pragma("temp_store", { simple: true }) ?? 0);
+    this.db.pragma("temp_store = FILE");
+    try {
+      this.db.exec("VACUUM");
+    } finally {
+      this.db.pragma(`temp_store = ${prevTempStore}`);
+    }
+  }
+
   /**
    * Reclaims unused SQLite pages and truncates the WAL file.
    * In-memory databases are skipped.
@@ -1104,7 +1118,7 @@ export class DocumentStore {
       const shouldVacuum = allowVacuum && (force || freelistCount > 0);
 
       if (shouldVacuum) {
-        this.db.exec("VACUUM");
+        this.vacuumWithFileTempStore();
         this.db.pragma("journal_mode = WAL");
         this.db.pragma("wal_autocheckpoint = 1000");
         this.db.pragma("wal_checkpoint(TRUNCATE)");