From 02ac871a71d767be46ac2f32bf462b03a58e8105 Mon Sep 17 00:00:00 2001 From: Luke Herbold Date: Tue, 10 Nov 2026 16:07:36 -0800 Subject: [PATCH] The sql.js connection keeps its prepared statements, so an ingest compiles each insert once rather than once per row Co-authored-by: Cursor --- src/sqljs/sqljs_connection.test.ts | 67 ++++++++++++++++++++++++++++++ src/sqljs/sqljs_connection.ts | 30 +++++++++++-- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/src/sqljs/sqljs_connection.test.ts b/src/sqljs/sqljs_connection.test.ts index ad52132e..db14ecd8 100644 --- a/src/sqljs/sqljs_connection.test.ts +++ b/src/sqljs/sqljs_connection.test.ts @@ -35,6 +35,29 @@ beforeAll(async () => { SQL = await factory({ locateFile: (file) => path.join(dist, file) }); }); +/** sql.js, counting how often each statement is prepared and how many statements are freed. */ +function countingModule() { + const prepares = new Map(); + let frees = 0; + const module = { + Database: class { + private readonly db = new SQL.Database(); + + prepare(sql: string) { + prepares.set(sql, (prepares.get(sql) ?? 0) + 1); + const statement = this.db.prepare(sql); + const free = statement.free.bind(statement); + statement.free = () => { + frees += 1; + free(); + }; + return statement; + } + }, + } as unknown as SqlJsModule; + return { module, prepared: (sql: string) => prepares.get(sql) ?? 0, freed: () => frees }; +} + function itemStore() { return defineSqliteStore({ name: 'items_store', @@ -84,6 +107,50 @@ describe('sql.js on the web', () => { expect(conn.execute('SELECT count(*) AS n FROM t;').rows?._array).toEqual([{ n: 2 }]); }); + it('prepares a statement once, however many times it runs', () => { + const counted = countingModule(); + const conn = openSqlJsConnection(counted.module); + conn.execute('CREATE TABLE t (x INTEGER);'); + + for (let x = 0; x < 50; x++) conn.execute('INSERT INTO t VALUES (?);', [x]); + + expect(counted.prepared('INSERT INTO t VALUES (?);')).toBe(1); + expect(conn.execute('SELECT count(*) AS n FROM t;').rows?._array).toEqual([{ n: 50 }]); + }); + + it('runs a statement again after it failed partway', () => { + const conn = openSqlJsConnection(SQL); + conn.execute('CREATE TABLE t (x INTEGER PRIMARY KEY);'); + conn.execute('INSERT INTO t VALUES (?);', [1]); + + expect(() => conn.execute('INSERT INTO t VALUES (?);', [1])).toThrow(); + conn.execute('INSERT INTO t VALUES (?);', [2]); + expect(conn.execute('SELECT x FROM t ORDER BY x;').rows?._array).toEqual([{ x: 1 }, { x: 2 }]); + }); + + it('leaves no statement holding a table, so the table can still be dropped and rebuilt', () => { + const conn = openSqlJsConnection(SQL); + conn.execute('CREATE TABLE t (x INTEGER);'); + conn.execute('INSERT INTO t VALUES (?);', [1]); + conn.execute('SELECT x FROM t;'); + + conn.execute('DROP TABLE t;'); + conn.execute('CREATE TABLE t (x INTEGER, y TEXT);'); + conn.execute('INSERT INTO t VALUES (?, ?);', [2, 'two']); + + expect(conn.execute('SELECT x, y FROM t;').rows?._array).toEqual([{ x: 2, y: 'two' }]); + }); + + it('keeps a bounded number of statements, freeing the least recently used', () => { + const counted = countingModule(); + const conn = openSqlJsConnection(counted.module); + conn.execute('CREATE TABLE t (x INTEGER);'); + + for (let x = 0; x < 300; x++) conn.execute(`SELECT ${x} AS n FROM t;`); + + expect(counted.freed()).toBe(300 + 1 - 256); + }); + it('reports a bind that fails and leaves the store reading empty, rather than taking the page down', () => { const captureException = jest.fn(); configureCellar({ errors: { captureException, captureMessage: jest.fn() } }); diff --git a/src/sqljs/sqljs_connection.ts b/src/sqljs/sqljs_connection.ts index 62590f39..8f3f3594 100644 --- a/src/sqljs/sqljs_connection.ts +++ b/src/sqljs/sqljs_connection.ts @@ -16,6 +16,8 @@ interface SqlJsStatement { step(): boolean; /** The current row, as an object keyed by column. */ getAsObject(): unknown; + /** Ends the statement's run, so it can be bound and stepped again. */ + reset(): void; /** Frees the statement. */ free(): void; } @@ -38,22 +40,44 @@ interface BindableStore { bindSqlite: (conn: SqliteConnection, options?: BindOptions) => void; } +/** Prepared statements kept per database; a store runs a few dozen distinct ones, each of them over and over. */ +const STATEMENT_CACHE_MAX = 256; + /** * Opens a new in-memory sql.js database as a {@linkcode SqliteConnection}. sql.js is synchronous, so the async methods - * wrap the sync ones. + * wrap the sync ones. Statements are prepared once and kept, since an ingest runs the same insert once per row. */ export function openSqlJsConnection(SQL: SqlJsModule): SqliteConnection { const db = new SQL.Database(); + const statements = new Map(); + + /** The statement for `sql`, most recently used last, so the oldest is the one evicted. */ + const prepared = (sql: string): SqlJsStatement => { + let statement = statements.get(sql); + if (statement) { + statements.delete(sql); + } else { + statement = db.prepare(sql); + if (statements.size >= STATEMENT_CACHE_MAX) { + const [oldest, evicted] = statements.entries().next().value as [string, SqlJsStatement]; + statements.delete(oldest); + evicted.free(); + } + } + statements.set(sql, statement); + return statement; + }; const run = (sql: string, params?: ReadonlyArray): QueryExecResult => { - const statement = db.prepare(sql); + const statement = prepared(sql); try { statement.bind((params ?? []).map((param) => (param === undefined ? null : param))); const rows: unknown[] = []; while (statement.step()) rows.push(statement.getAsObject()); return { rows: { _array: rows } }; } finally { - statement.free(); + // A statement left mid-run holds its table, and SQLite refuses to drop or alter a table a statement holds. + statement.reset(); } };