Web: add an opt-in storageBucket option for OPFS databases

.changeset/storage-bucket-option.md

@@ -0,0 +1,5 @@
+---
+'@powersync/web': minor
+---
+
+Add the `storageBucket` option, to keep an OPFS database in a Storage Bucket instead of the default bucket of the origin private file system.

packages/web/src/db/adapters/options.ts

@@ -107,6 +107,31 @@ export interface WebSpecificOpenOptions {
    * Defaults to 0 (disabling the cache).
    */
   preparedStatementsCache?: number;
+
+  /**
+   * The name of a [Storage Bucket](https://wicg.github.io/storage-buckets/) to keep the database files in,
+   * instead of the default bucket of the origin private file system.
+   *
+   * Only supported by the OPFS-based VFS implementations: {@link WASQLiteVFS.OPFSCoopSyncVFS},
+   * {@link WASQLiteVFS.OPFSWriteAheadVFS} and {@link WASQLiteVFS.AccessHandlePoolVFS}. Setting it in a browser
+   * without the Storage Buckets API is a configuration error, so set this option only when
+   * `'storageBuckets' in navigator`.
+   *
+   * The bucket is opened by name, with the browser's default options. Persistent storage is granted per origin by
+   * `navigator.storage.persist()`, and that call marks only the default bucket. For a named bucket, call
+   * `navigator.storage.persist()` first and then `persist()` on the bucket, which your app can open by the same name
+   * with `navigator.storageBuckets.open(name)`; the bucket-level call takes effect only while the origin has the
+   * permission. Bucket names have lowercase letters, digits, `-` and `_`, start with a letter or a digit, and have at
+   * most 63 characters.
+   *
+   * A database in a bucket is a different database from one of the same name in the default bucket: setting this
+   * option for an existing database starts with an empty database, and the files in the default bucket are not
+   * moved. The SDK's cross-tab locks and channels are named after `dbFilename` alone, so give the database a name no
+   * other database of the app uses, in any bucket.
+   *
+   * Defaults to undefined, which uses the default bucket.
+   */
+  storageBucket?: string;
 }
 
 export interface ResolvedWebSQLOpenOptions extends SQLOpenOptions, WebSpecificOpenOptions {}

packages/web/src/db/adapters/resolveAndValidateOptions.ts

@@ -1,6 +1,6 @@
 import { LogLevels } from '@powersync/common';
 import { TemporaryStorageOption, WebSpecificOpenOptions } from './options.js';
-import { vfsRequiresDedicatedWorkers, WASQLiteVFS } from './wa-sqlite/vfs.js';
+import { vfsRequiresDedicatedWorkers, vfsSupportsStorageBuckets, WASQLiteVFS } from './wa-sqlite/vfs.js';
 
 /**
  * The maximum length of a db filename we support.
@@ -45,5 +45,24 @@ export function resolveAndValidateOptions<And = {}>(
     );
   }
 
+  if (resolved.storageBucket != null) {
+    if (!vfsSupportsStorageBuckets(resolved.vfs)) {
+      throw new Error(
+        `Invalid configuration: The 'storageBucket' option needs an OPFS-based VFS, which ${resolved.vfs} is not.`
+      );
+    }
+    // The naming rules of https://wicg.github.io/storage-buckets/, with the length Chromium accepts.
+    if (!/^[a-z0-9][a-z0-9_-]{0,62}$/.test(resolved.storageBucket)) {
+      throw new Error(
+        `Invalid configuration: '${resolved.storageBucket}' is not a valid Storage Bucket name. A name has lowercase letters, digits, '-' and '_', starts with a letter or a digit, and has at most 63 characters.`
+      );
+    }
+    if (typeof navigator !== 'undefined' && !('storageBuckets' in navigator)) {
+      throw new Error(
+        `Invalid configuration: The 'storageBucket' option needs the Storage Buckets API (navigator.storageBuckets), which this browser does not have. Set the option only when 'storageBuckets' in navigator.`
+      );
+    }
+  }
+
   return resolved;
 }

packages/web/src/db/adapters/wa-sqlite/RawSqliteConnection.ts

@@ -30,6 +30,10 @@ export interface RawWaSqliteDatabaseOptions {
    */
   preparedStatementsCache: number;
   additionalInitializationFunctions?: readonly string[];
+  /**
+   * The name of a Storage Bucket to keep the database files in, for the OPFS-based VFS implementations.
+   */
+  storageBucket?: string;
 }
 
 /**

packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts

@@ -76,7 +76,8 @@ export class WASQLiteOpenFactory implements SQLOpenFactory {
       temporaryStorage,
       cacheSizeKb,
       preparedStatementsCache,
-      additionalInitializationFunctions
+      additionalInitializationFunctions,
+      storageBucket
     } = this.options;
 
     if (!enableMultiTabs) {
@@ -97,7 +98,8 @@ export class WASQLiteOpenFactory implements SQLOpenFactory {
         cacheSizeKb,
         // TODO: Enable prepared statement cache by default?
         preparedStatementsCache: preparedStatementsCache ?? 0,
-        additionalInitializationFunctions
+        additionalInitializationFunctions,
+        storageBucket
       };
     }
 

packages/web/src/db/adapters/wa-sqlite/vfs.ts

@@ -1,5 +1,6 @@
 import type * as SQLite from '@journeyapps/wa-sqlite';
 import { RawWaSqliteDatabaseOptions } from './RawSqliteConnection.js';
+import type { WebSpecificOpenOptions } from '../options.js';
 
 /**
  * List of currently tested virtual filesystems
@@ -32,6 +33,51 @@ export function vfsRequiresDedicatedWorkers(vfs: WASQLiteVFS) {
   return vfs != WASQLiteVFS.IDBBatchAtomicVFS && vfs != WASQLiteVFS.InMemoryVfs;
 }
 
+/**
+ * Whether the VFS can keep its files in a Storage Bucket, see {@link WebSpecificOpenOptions.storageBucket}.
+ */
+export function vfsSupportsStorageBuckets(vfs: WASQLiteVFS) {
+  return (
+    vfs == WASQLiteVFS.OPFSCoopSyncVFS || vfs == WASQLiteVFS.OPFSWriteAheadVFS || vfs == WASQLiteVFS.AccessHandlePoolVFS
+  );
+}
+
+/**
+ * The part of the Storage Buckets API this module uses. The API is not in the TypeScript DOM library yet.
+ */
+export interface StorageBucketManagerLike {
+  open(name: string): Promise<{ getDirectory(): Promise<FileSystemDirectoryHandle> }>;
+}
+
+/**
+ * Options for an OPFS-based VFS that keep its files in the named Storage Bucket, see
+ * {@link WebSpecificOpenOptions.storageBucket}. Without a bucket name, no options: the VFS uses the root of the
+ * origin private file system.
+ *
+ * @internal
+ */
+export function vfsOptionsForStorageBucket(
+  storageBucket: string | undefined,
+  storageBuckets: StorageBucketManagerLike | null | undefined = (navigator as any).storageBuckets
+): { getRoot?: () => Promise<FileSystemDirectoryHandle>; prefix?: string } {
+  if (storageBucket == null) {
+    return {};
+  }
+  if (storageBuckets == null) {
+    throw new Error(
+      `The 'storageBucket' option needs the Storage Buckets API (navigator.storageBuckets), which this browser does not have. Check for it before setting the option.`
+    );
+  }
+  return {
+    // Called by the VFS each time it needs its root directory. Opening the bucket again creates it when the
+    // browser deleted it with the rest of the site's data.
+    getRoot: () => storageBuckets.open(storageBucket).then((bucket) => bucket.getDirectory()),
+    // Keeps the locks and channels of a file apart from those of a file of the same name in another bucket.
+    // Bucket names cannot contain ':', so the names of two buckets never overlap.
+    prefix: `${storageBucket}:`
+  };
+}
+
 /**
  * @internal
  */
@@ -68,7 +114,8 @@ async function syncModuleFactory(encryptionKey: string | undefined): Promise<SQL
 export async function loadModuleAndVfs({
   vfs,
   filename,
-  encryptionKey
+  encryptionKey,
+  storageBucket
 }: RawWaSqliteDatabaseOptions): Promise<{ module: SQLiteModule; vfs: SQLiteVFS }> {
   let moduleFactory = syncModuleFactory;
   let resolveVfs: (module: any) => Promise<SQLiteVFS>;
@@ -86,19 +133,19 @@ export async function loadModuleAndVfs({
     case WASQLiteVFS.AccessHandlePoolVFS: {
       // @ts-expect-error The types for this import are missing upstream
       const { AccessHandlePoolVFS } = await import('@journeyapps/wa-sqlite/src/examples/AccessHandlePoolVFS.js');
-      resolveVfs = (module) => AccessHandlePoolVFS.create(filename, module);
+      resolveVfs = (module) => AccessHandlePoolVFS.create(filename, module, vfsOptionsForStorageBucket(storageBucket));
       break;
     }
     case WASQLiteVFS.OPFSCoopSyncVFS: {
       // @ts-expect-error The types for this import are missing upstream
       const { OPFSCoopSyncVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSCoopSyncVFS.js');
-      resolveVfs = (module) => OPFSCoopSyncVFS.create(filename, module);
+      resolveVfs = (module) => OPFSCoopSyncVFS.create(filename, module, vfsOptionsForStorageBucket(storageBucket));
       break;
     }
     case WASQLiteVFS.OPFSWriteAheadVFS: {
       // @ts-expect-error The types for this import are missing upstream
       const { OPFSWriteAheadVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSWriteAheadVFS.js');
-      resolveVfs = (module) => OPFSWriteAheadVFS.create(filename, module, {});
+      resolveVfs = (module) => OPFSWriteAheadVFS.create(filename, module, vfsOptionsForStorageBucket(storageBucket));
       break;
     }
     case WASQLiteVFS.InMemoryVfs: {

packages/web/tests/storage_bucket.test.ts

@@ -0,0 +1,125 @@
+import { PowerSyncDatabase, WASQLiteVFS } from '@powersync/web';
+import { v4 as uuid } from 'uuid';
+import { describe, expect, it, vi } from 'vitest';
+import { StorageBucketManagerLike, vfsOptionsForStorageBucket } from '../src/db/adapters/wa-sqlite/vfs.js';
+import { defaultTestLogger } from './utils/logger.js';
+import { TEST_SCHEMA } from './utils/test-schema.js';
+import { generateTestDb } from './utils/testDb.js';
+
+const BUCKET = 'powersync-test';
+
+// Reads through the writer connection. A reader connection of OPFSWriteAheadVFS can briefly miss a write that just
+// finished, with or without a bucket, so a read straight after a write would make these tests flaky.
+function descriptions(db: PowerSyncDatabase): Promise<{ description: string }[]> {
+  return db.writeTransaction((tx) => tx.getAll<{ description: string }>('SELECT description FROM assets'));
+}
+
+async function namesIn(directory: FileSystemDirectoryHandle): Promise<string[]> {
+  const names: string[] = [];
+  for await (const name of (directory as any).keys()) {
+    names.push(name);
+  }
+  return names;
+}
+
+async function bucketDirectory(): Promise<FileSystemDirectoryHandle> {
+  const bucket = await (navigator as any).storageBuckets.open(BUCKET);
+  return bucket.getDirectory();
+}
+
+describe('storageBucket', { sequential: true }, () => {
+  const supportedVfs = [WASQLiteVFS.AccessHandlePoolVFS, WASQLiteVFS.OPFSCoopSyncVFS, WASQLiteVFS.OPFSWriteAheadVFS];
+
+  for (const vfs of supportedVfs) {
+    describe.skipIf(!('storageBuckets' in navigator))(vfs, () => {
+      it('keeps the database files in the bucket', async () => {
+        const dbFilename = `${uuid()}.db`;
+        const db = generateTestDb({
+          schema: TEST_SCHEMA,
+          logger: defaultTestLogger,
+          database: { dbFilename, vfs, storageBucket: BUCKET }
+        });
+        await db.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['in the bucket']);
+        expect(await descriptions(db)).toEqual([{ description: 'in the bucket' }]);
+
+        // AccessHandlePoolVFS keeps its files in a directory named after the database.
+        expect(await namesIn(await bucketDirectory())).toContain(dbFilename);
+        expect(await namesIn(await navigator.storage.getDirectory())).not.toContain(dbFilename);
+      });
+
+      it('keeps apart from a database of the same name in the default bucket', async () => {
+        // The same name on purpose, although the option's documentation advises against it: this shows that the
+        // files and the VFS locks of the two databases stay apart.
+        const dbFilename = `${uuid()}.db`;
+        const inBucket = generateTestDb({
+          schema: TEST_SCHEMA,
+          logger: defaultTestLogger,
+          database: { dbFilename, vfs, storageBucket: BUCKET }
+        });
+        const inDefault = generateTestDb({
+          schema: TEST_SCHEMA,
+          logger: defaultTestLogger,
+          database: { dbFilename, vfs }
+        });
+
+        await inBucket.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['bucket']);
+        await inDefault.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['default']);
+        await inDefault.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['default']);
+
+        expect(await descriptions(inBucket)).toEqual([{ description: 'bucket' }]);
+        expect(await descriptions(inDefault)).toEqual([{ description: 'default' }, { description: 'default' }]);
+      });
+    });
+  }
+
+  describe('vfsOptionsForStorageBucket', () => {
+    it('gives the VFS no options without a bucket name', () => {
+      expect(vfsOptionsForStorageBucket(undefined)).toEqual({});
+      expect(vfsOptionsForStorageBucket(undefined, null)).toEqual({});
+    });
+
+    it('opens the bucket for the root directory and prefixes lock names with the bucket name', async () => {
+      const directory = {} as FileSystemDirectoryHandle;
+      const bucket = { getDirectory: vi.fn(async () => directory) };
+      const buckets: StorageBucketManagerLike = { open: vi.fn(async () => bucket) };
+
+      const options = vfsOptionsForStorageBucket('app-data', buckets);
+      expect(options.prefix).toBe('app-data:');
+      expect(buckets.open).not.toHaveBeenCalled();
+
+      // Each call opens the bucket again, so a bucket the browser deleted is created again.
+      expect(await options.getRoot!()).toBe(directory);
+      expect(await options.getRoot!()).toBe(directory);
+      expect(buckets.open).toHaveBeenCalledTimes(2);
+      expect(buckets.open).toHaveBeenCalledWith('app-data');
+    });
+
+    it('fails without the Storage Buckets API', () => {
+      expect(() => vfsOptionsForStorageBucket('app-data', null)).toThrow(
+        "The 'storageBucket' option needs the Storage Buckets API"
+      );
+    });
+  });
+
+  it('rejects a bucket name the Storage Buckets API would not accept', () => {
+    expect(
+      () =>
+        new PowerSyncDatabase({
+          schema: TEST_SCHEMA,
+          logger: defaultTestLogger,
+          database: { dbFilename: `${uuid()}.db`, vfs: WASQLiteVFS.OPFSWriteAheadVFS, storageBucket: 'Not-Valid' }
+        })
+    ).toThrow('is not a valid Storage Bucket name');
+  });
+
+  it('rejects a VFS that does not keep its files in OPFS', () => {
+    expect(
+      () =>
+        new PowerSyncDatabase({
+          schema: TEST_SCHEMA,
+          logger: defaultTestLogger,
+          database: { dbFilename: `${uuid()}.db`, vfs: WASQLiteVFS.IDBBatchAtomicVFS, storageBucket: BUCKET }
+        })
+    ).toThrow("The 'storageBucket' option needs an OPFS-based VFS");
+  });
+});