diff --git a/.changeset/storage-bucket-option.md b/.changeset/storage-bucket-option.md new file mode 100644 index 000000000..583d9c05c --- /dev/null +++ b/.changeset/storage-bucket-option.md @@ -0,0 +1,5 @@ +--- +'@powersync/web': minor +--- + +Add the `storageBucket` option, to keep an OPFS database in a Storage Bucket instead of the default bucket of the origin private file system. diff --git a/packages/web/src/db/adapters/options.ts b/packages/web/src/db/adapters/options.ts index 4134b079d..2762b0588 100644 --- a/packages/web/src/db/adapters/options.ts +++ b/packages/web/src/db/adapters/options.ts @@ -107,6 +107,27 @@ export interface WebSpecificOpenOptions { * Defaults to 0 (disabling the cache). */ preparedStatementsCache?: number; + + /** + * The name of a [Storage Bucket](https://wicg.github.io/storage-buckets/) to keep the database files in, + * instead of the default bucket of the origin private file system. + * + * Only supported by the OPFS-based VFS implementations: {@link WASQLiteVFS.OPFSCoopSyncVFS}, + * {@link WASQLiteVFS.OPFSWriteAheadVFS} and {@link WASQLiteVFS.AccessHandlePoolVFS}. Opening the database fails in + * a browser without the Storage Buckets API, so set this option only when `'storageBuckets' in navigator`. + * + * The SDK forwards the name to `navigator.storageBuckets.open()` unchanged, so the browser reports a name it does not + * accept. The bucket is opened by name, with the browser's default options. To configure options for the bucket, + * call and await `navigator.storageBuckets.open()` before opening a PowerSync database. + * + * A database in a bucket is a different database from one of the same name in the default bucket: setting this + * option for an existing database starts with an empty database, and the files in the default bucket are not + * moved. The SDK groups database resources by database name, without taking the storage bucket into account. To + * avoid lock contention and wrong update notifications, use unique database names even across buckets. + * + * Uses the default bucket when no option is given. + */ + storageBucket?: string; } export interface ResolvedWebSQLOpenOptions extends SQLOpenOptions, WebSpecificOpenOptions {} diff --git a/packages/web/src/db/adapters/resolveAndValidateOptions.ts b/packages/web/src/db/adapters/resolveAndValidateOptions.ts index 5869a7b64..139f2b3dd 100644 --- a/packages/web/src/db/adapters/resolveAndValidateOptions.ts +++ b/packages/web/src/db/adapters/resolveAndValidateOptions.ts @@ -1,6 +1,6 @@ import { LogLevels } from '@powersync/common'; import { TemporaryStorageOption, WebSpecificOpenOptions } from './options.js'; -import { vfsRequiresDedicatedWorkers, WASQLiteVFS } from './wa-sqlite/vfs.js'; +import { vfsRequiresDedicatedWorkers, vfsSupportsStorageBuckets, WASQLiteVFS } from './wa-sqlite/vfs.js'; /** * The maximum length of a db filename we support. @@ -45,5 +45,13 @@ export function resolveAndValidateOptions( ); } + if (resolved.storageBucket != null) { + if (!vfsSupportsStorageBuckets(resolved.vfs)) { + throw new Error( + `Invalid configuration: The 'storageBucket' option needs an OPFS-based VFS, which ${resolved.vfs} is not.` + ); + } + } + return resolved; } diff --git a/packages/web/src/db/adapters/wa-sqlite/RawSqliteConnection.ts b/packages/web/src/db/adapters/wa-sqlite/RawSqliteConnection.ts index 26bd49cc4..9931836e5 100644 --- a/packages/web/src/db/adapters/wa-sqlite/RawSqliteConnection.ts +++ b/packages/web/src/db/adapters/wa-sqlite/RawSqliteConnection.ts @@ -30,6 +30,10 @@ export interface RawWaSqliteDatabaseOptions { */ preparedStatementsCache: number; additionalInitializationFunctions?: readonly string[]; + /** + * The name of a Storage Bucket to keep the database files in, for the OPFS-based VFS implementations. + */ + storageBucket?: string; } /** diff --git a/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts b/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts index 1295b1ba4..399a5fc35 100644 --- a/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts +++ b/packages/web/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.ts @@ -76,7 +76,8 @@ export class WASQLiteOpenFactory implements SQLOpenFactory { temporaryStorage, cacheSizeKb, preparedStatementsCache, - additionalInitializationFunctions + additionalInitializationFunctions, + storageBucket } = this.options; if (!enableMultiTabs) { @@ -97,7 +98,8 @@ export class WASQLiteOpenFactory implements SQLOpenFactory { cacheSizeKb, // TODO: Enable prepared statement cache by default? preparedStatementsCache: preparedStatementsCache ?? 0, - additionalInitializationFunctions + additionalInitializationFunctions, + storageBucket }; } diff --git a/packages/web/src/db/adapters/wa-sqlite/vfs.ts b/packages/web/src/db/adapters/wa-sqlite/vfs.ts index df133aa61..a3a6371cb 100644 --- a/packages/web/src/db/adapters/wa-sqlite/vfs.ts +++ b/packages/web/src/db/adapters/wa-sqlite/vfs.ts @@ -1,5 +1,6 @@ import type * as SQLite from '@journeyapps/wa-sqlite'; import { RawWaSqliteDatabaseOptions } from './RawSqliteConnection.js'; +import type { WebSpecificOpenOptions } from '../options.js'; /** * List of currently tested virtual filesystems @@ -32,6 +33,51 @@ export function vfsRequiresDedicatedWorkers(vfs: WASQLiteVFS) { return vfs != WASQLiteVFS.IDBBatchAtomicVFS && vfs != WASQLiteVFS.InMemoryVfs; } +/** + * Whether the VFS can keep its files in a Storage Bucket, see {@link WebSpecificOpenOptions.storageBucket}. + */ +export function vfsSupportsStorageBuckets(vfs: WASQLiteVFS) { + return ( + vfs == WASQLiteVFS.OPFSCoopSyncVFS || vfs == WASQLiteVFS.OPFSWriteAheadVFS || vfs == WASQLiteVFS.AccessHandlePoolVFS + ); +} + +/** + * The part of the Storage Buckets API this module uses. The API is not in the TypeScript DOM library yet. + */ +export interface StorageBucketManagerLike { + open(name: string): Promise<{ getDirectory(): Promise }>; +} + +/** + * Options for an OPFS-based VFS that keep its files in the named Storage Bucket, see + * {@link WebSpecificOpenOptions.storageBucket}. Without a bucket name, no options: the VFS uses the root of the + * origin private file system. + * + * @internal + */ +export function vfsOptionsForStorageBucket( + storageBucket: string | undefined, + storageBuckets: StorageBucketManagerLike | null | undefined = (navigator as any).storageBuckets +): { getRoot?: () => Promise; prefix?: string } { + if (storageBucket == null) { + return {}; + } + if (storageBuckets == null) { + throw new Error( + `The 'storageBucket' option needs the Storage Buckets API (navigator.storageBuckets), which this browser does not have. Check for it before setting the option.` + ); + } + return { + // Called by the VFS each time it needs its root directory. Opening the bucket again creates it when the + // browser deleted it with the rest of the site's data. + getRoot: () => storageBuckets.open(storageBucket).then((bucket) => bucket.getDirectory()), + // Keeps the locks and channels of a file apart from those of a file of the same name in another bucket. + // Bucket names cannot contain ':', so the names of two buckets never overlap. + prefix: `${storageBucket}:` + }; +} + /** * @internal */ @@ -68,7 +114,8 @@ async function syncModuleFactory(encryptionKey: string | undefined): Promise { let moduleFactory = syncModuleFactory; let resolveVfs: (module: any) => Promise; @@ -86,19 +133,19 @@ export async function loadModuleAndVfs({ case WASQLiteVFS.AccessHandlePoolVFS: { // @ts-expect-error The types for this import are missing upstream const { AccessHandlePoolVFS } = await import('@journeyapps/wa-sqlite/src/examples/AccessHandlePoolVFS.js'); - resolveVfs = (module) => AccessHandlePoolVFS.create(filename, module); + resolveVfs = (module) => AccessHandlePoolVFS.create(filename, module, vfsOptionsForStorageBucket(storageBucket)); break; } case WASQLiteVFS.OPFSCoopSyncVFS: { // @ts-expect-error The types for this import are missing upstream const { OPFSCoopSyncVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSCoopSyncVFS.js'); - resolveVfs = (module) => OPFSCoopSyncVFS.create(filename, module); + resolveVfs = (module) => OPFSCoopSyncVFS.create(filename, module, vfsOptionsForStorageBucket(storageBucket)); break; } case WASQLiteVFS.OPFSWriteAheadVFS: { // @ts-expect-error The types for this import are missing upstream const { OPFSWriteAheadVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSWriteAheadVFS.js'); - resolveVfs = (module) => OPFSWriteAheadVFS.create(filename, module, {}); + resolveVfs = (module) => OPFSWriteAheadVFS.create(filename, module, vfsOptionsForStorageBucket(storageBucket)); break; } case WASQLiteVFS.InMemoryVfs: { diff --git a/packages/web/tests/storage_bucket.test.ts b/packages/web/tests/storage_bucket.test.ts new file mode 100644 index 000000000..818244d90 --- /dev/null +++ b/packages/web/tests/storage_bucket.test.ts @@ -0,0 +1,114 @@ +import { PowerSyncDatabase, WASQLiteVFS } from '@powersync/web'; +import { v4 as uuid } from 'uuid'; +import { describe, expect, it, vi } from 'vitest'; +import { StorageBucketManagerLike, vfsOptionsForStorageBucket } from '../src/db/adapters/wa-sqlite/vfs.js'; +import { defaultTestLogger } from './utils/logger.js'; +import { TEST_SCHEMA } from './utils/test-schema.js'; +import { generateTestDb } from './utils/testDb.js'; + +const BUCKET = 'powersync-test'; + +// Reads through the writer connection. A reader connection of OPFSWriteAheadVFS can briefly miss a write that just +// finished, with or without a bucket, so a read straight after a write would make these tests flaky. +function descriptions(db: PowerSyncDatabase): Promise<{ description: string }[]> { + return db.writeTransaction((tx) => tx.getAll<{ description: string }>('SELECT description FROM assets')); +} + +async function namesIn(directory: FileSystemDirectoryHandle): Promise { + const names: string[] = []; + for await (const name of (directory as any).keys()) { + names.push(name); + } + return names; +} + +async function bucketDirectory(): Promise { + const bucket = await (navigator as any).storageBuckets.open(BUCKET); + return bucket.getDirectory(); +} + +describe('storageBucket', { sequential: true }, () => { + const supportedVfs = [WASQLiteVFS.AccessHandlePoolVFS, WASQLiteVFS.OPFSCoopSyncVFS, WASQLiteVFS.OPFSWriteAheadVFS]; + + for (const vfs of supportedVfs) { + describe.skipIf(!('storageBuckets' in navigator))(vfs, () => { + it('keeps the database files in the bucket', async () => { + const dbFilename = `${uuid()}.db`; + const db = generateTestDb({ + schema: TEST_SCHEMA, + logger: defaultTestLogger, + database: { dbFilename, vfs, storageBucket: BUCKET } + }); + await db.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['in the bucket']); + expect(await descriptions(db)).toEqual([{ description: 'in the bucket' }]); + + // AccessHandlePoolVFS keeps its files in a directory named after the database. + expect(await namesIn(await bucketDirectory())).toContain(dbFilename); + expect(await namesIn(await navigator.storage.getDirectory())).not.toContain(dbFilename); + }); + + it('keeps apart from a database of the same name in the default bucket', async () => { + // The same name on purpose, although the option's documentation advises against it: this shows that the + // files and the VFS locks of the two databases stay apart. + const dbFilename = `${uuid()}.db`; + const inBucket = generateTestDb({ + schema: TEST_SCHEMA, + logger: defaultTestLogger, + database: { dbFilename, vfs, storageBucket: BUCKET } + }); + const inDefault = generateTestDb({ + schema: TEST_SCHEMA, + logger: defaultTestLogger, + database: { dbFilename, vfs } + }); + + await inBucket.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['bucket']); + await inDefault.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['default']); + await inDefault.execute('INSERT INTO assets(id, description) VALUES(uuid(), ?)', ['default']); + + expect(await descriptions(inBucket)).toEqual([{ description: 'bucket' }]); + expect(await descriptions(inDefault)).toEqual([{ description: 'default' }, { description: 'default' }]); + }); + }); + } + + describe('vfsOptionsForStorageBucket', () => { + it('gives the VFS no options without a bucket name', () => { + expect(vfsOptionsForStorageBucket(undefined)).toEqual({}); + expect(vfsOptionsForStorageBucket(undefined, null)).toEqual({}); + }); + + it('opens the bucket for the root directory and prefixes lock names with the bucket name', async () => { + const directory = {} as FileSystemDirectoryHandle; + const bucket = { getDirectory: vi.fn(async () => directory) }; + const buckets: StorageBucketManagerLike = { open: vi.fn(async () => bucket) }; + + const options = vfsOptionsForStorageBucket('app-data', buckets); + expect(options.prefix).toBe('app-data:'); + expect(buckets.open).not.toHaveBeenCalled(); + + // Each call opens the bucket again, so a bucket the browser deleted is created again. + expect(await options.getRoot!()).toBe(directory); + expect(await options.getRoot!()).toBe(directory); + expect(buckets.open).toHaveBeenCalledTimes(2); + expect(buckets.open).toHaveBeenCalledWith('app-data'); + }); + + it('fails without the Storage Buckets API', () => { + expect(() => vfsOptionsForStorageBucket('app-data', null)).toThrow( + "The 'storageBucket' option needs the Storage Buckets API" + ); + }); + }); + + it('rejects a VFS that does not keep its files in OPFS', () => { + expect( + () => + new PowerSyncDatabase({ + schema: TEST_SCHEMA, + logger: defaultTestLogger, + database: { dbFilename: `${uuid()}.db`, vfs: WASQLiteVFS.IDBBatchAtomicVFS, storageBucket: BUCKET } + }) + ).toThrow("The 'storageBucket' option needs an OPFS-based VFS"); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c73600bc3..fb21685d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,8 +7,8 @@ settings: catalogs: default: '@journeyapps/wa-sqlite': - specifier: 2.0.8 - version: 2.0.8 + specifier: 2.0.10 + version: 2.0.10 '@microsoft/api-extractor': specifier: ^7.59.0 version: 7.59.0 @@ -411,7 +411,7 @@ importers: devDependencies: '@journeyapps/wa-sqlite': specifier: 'catalog:' - version: 2.0.8 + version: 2.0.10 '@powersync/web': specifier: workspace:* version: link:../web @@ -436,7 +436,7 @@ importers: devDependencies: '@journeyapps/wa-sqlite': specifier: 'catalog:' - version: 2.0.8 + version: 2.0.10 '@powersync/web': specifier: workspace:* version: link:../web @@ -549,7 +549,7 @@ importers: devDependencies: '@journeyapps/wa-sqlite': specifier: 'catalog:' - version: 2.0.8 + version: 2.0.10 '@nuxt/module-builder': specifier: ^1.0.2 version: 1.0.2(@nuxt/cli@3.34.0(@nuxt/schema@4.3.1)(cac@6.7.14)(commander@12.1.0)(magicast@0.5.2)(supports-color@10.2.2))(@vue/compiler-core@3.5.43)(esbuild@0.27.3)(typescript@6.0.3)(vue-tsc@3.2.5(typescript@6.0.3))(vue@3.5.30(typescript@6.0.3)) @@ -801,7 +801,7 @@ importers: dependencies: '@journeyapps/wa-sqlite': specifier: 'catalog:' - version: 2.0.8 + version: 2.0.10 '@powersync/common': specifier: workspace:* version: link:../common @@ -896,7 +896,7 @@ importers: dependencies: '@journeyapps/wa-sqlite': specifier: 'catalog:' - version: 2.0.8 + version: 2.0.10 '@monaco-editor/react': specifier: ^4.7.0 version: 4.7.0(monaco-editor@0.55.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -4166,8 +4166,8 @@ packages: resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} - '@journeyapps/wa-sqlite@2.0.8': - resolution: {integrity: sha512-WRXEms2515S/0DFSWzmlggHu9hSJomhugDUO17rWkL6VpokV/hfjIDLs954Egx/eqrk3WADmiOMzOc7MuJbznQ==} + '@journeyapps/wa-sqlite@2.0.10': + resolution: {integrity: sha512-6Wo5ZijOc4/6M9u+MDsJdW8zDCdrct/p+7LcG7S7CT5YKu42j6lshMfUvgtGYMaZeuH7rpQHULZupPTQFqbS0w==} '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -23221,7 +23221,7 @@ snapshots: '@types/yargs': 17.0.35 chalk: 4.1.2 - '@journeyapps/wa-sqlite@2.0.8': {} + '@journeyapps/wa-sqlite@2.0.10': {} '@jridgewell/gen-mapping@0.3.13': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 69998543b..d9d4c7ac3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,7 +7,7 @@ packages: catalog: # We must depend on an exact version of wa-sqlite, as we want to update the core extension # (which can be breaking for SDKs) without a major revision. - '@journeyapps/wa-sqlite': 2.0.8 + '@journeyapps/wa-sqlite': 2.0.10 '@microsoft/api-extractor': ^7.59.0 '@nuxt/devtools-kit': ^3.2.3 '@nuxt/devtools-ui-kit': ^3.2.3