Add vector_index virtual table

README.md

@@ -114,6 +114,59 @@ LIMIT 10;
 
 Values outside the configured range are clamped silently. Calling `vector_quantize` or `vector_distance_q` without configuring a range returns a SQL error.
 
+## Vector Index
+
+The `vector_index` virtual table is a faster alternative to `ORDER BY vector_distance(...)` for large tables. It stores vectors in chunks of about 1 MiB and scans them in Go across `GOMAXPROCS` goroutines, instead of calling a SQL function for every row.
+
+```sql
+CREATE VIRTUAL TABLE docs_vec USING vector_index();     -- float32
+CREATE VIRTUAL TABLE docs_q USING vector_index(int8);   -- quantized, requires WithQuantRange
+
+INSERT INTO docs_vec (rowid, embedding) VALUES (1, vector_encode('[0.1, 0.2, 0.3]'));
+
+-- k nearest neighbors
+SELECT rowid, distance
+FROM docs_vec
+WHERE embedding MATCH vector_encode('[0.15, 0.25, 0.35]') AND k = 5;
+
+-- LIMIT also works when the query reads only docs_vec
+SELECT rowid, distance
+FROM docs_vec
+WHERE embedding MATCH vector_encode('[0.15, 0.25, 0.35]')
+ORDER BY distance
+LIMIT 5;
+
+-- joins need k, because SQLite does not pass LIMIT through a join
+SELECT d.content, v.distance
+FROM docs_vec v
+JOIN documents d ON d.id = v.rowid
+WHERE v.embedding MATCH vector_encode('[0.15, 0.25, 0.35]') AND v.k = 5
+ORDER BY v.distance;
+```
+
+`embedding` is written and matched as a float32 blob. int8 tables quantize on write and on search using the `WithQuantRange` range, and return `vector_distance_q` distances. `UPDATE` and `DELETE` work as on a normal table.
+
+### Chunk size
+
+By default each chunk holds about 1 MiB of vector data: 2048 float32 vectors at 128 dimensions, 170 at 1536. `chunk_size=N` sets the number of vectors per chunk when the table is created:
+
+```sql
+CREATE VIRTUAL TABLE docs_vec USING vector_index(float32, chunk_size=512);
+```
+
+Larger chunks mean fewer SQLite rows to read per search, but writes into a chunk slow down as it grows. Measured on Apple M3 Max at 128, 768, and 1536 dimensions:
+
+| Vector data per chunk | Insert, vs 1 MiB | Search p50, vs 1 MiB |
+|---|---|---|
+| 64 KiB | up to 6 µs per row faster; 10 µs slower for float32 at 768 dimensions | 14-133% slower |
+| 256 KiB | up to 5 µs per row faster | 10-46% slower for float32; int8 within 10% |
+| 1 MiB (default) | | |
+| 4 MiB | 4.5-16x slower | at most 4% faster |
+
+The chunk size cannot be changed after creation; copy the rows into a new table to change it.
+
+The table keeps its data in ordinary shadow tables (`docs_vec_info`, `docs_vec_chunks`, `docs_vec_data`, `docs_vec_rowids`), so writes follow SQLite transactions. Reopening a database with a different dimension or quantization range than the table was created with returns an error.
+
 ## Embedding
 
 Optional `vector_embed` function converts text to embeddings inside SQL. Provide an `Embedder` implementation via `WithEmbedder`:
@@ -167,7 +220,7 @@ Each row returned by `vector_chunk` has a `value` (the chunk text) and a `chunk_
 
 ## Design
 
-- **Brute-force scan**: search is a linear scan over all rows, appropriate for SQLite-scale datasets (thousands to low millions of vectors).
+- **Brute-force scan**: search is a linear scan over all rows, through the scalar functions or `vector_index`, appropriate for SQLite-scale datasets (thousands to low millions of vectors).
 - **Pure Go**: all vector math uses `encoding/binary` and `math` from the standard library.
 - **Single package**: everything lives in package `vector` at the module root. All internals are unexported.
 
@@ -193,7 +246,7 @@ BenchmarkL2DistanceQuantized/dim=1536  3041773     398.2 ns/op     0 B/op    0 a
 
 ### SIFT1M
 
-[SIFT1M](http://corpus-texmex.irisa.fr/) is a standard nearest-neighbor benchmark: 1,000,000 base vectors and 10,000 query vectors, 128 dimensions, L2 distance, with published ground-truth neighbors. The first 100 queries run single-threaded with k=100 against an on-disk database with default SQLite settings.
+[SIFT1M](http://corpus-texmex.irisa.fr/) is a standard nearest-neighbor benchmark: 1,000,000 base vectors and 10,000 query vectors, 128 dimensions, L2 distance, with published ground-truth neighbors. The first 100 queries run with k=100 against an on-disk database with default SQLite settings. Every implementation searches on one thread except `vector_index`, which uses all 14 cores of the test machine.
 
 ```
 curl -O ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz
@@ -206,8 +259,10 @@ Results on Apple M3 Max, macOS 15.7, Go 1.24.12. Build time is the time to inser
 
 | Implementation | Search | Build | p50 | QPS | Recall@10 | Recall@100 |
 |---|---|---|---|---|---|---|
-| go-sqlite-vector `vector_distance` | exact scan | 2.4s | 362 ms | 2.8 | 0.999 | 1.000 |
-| go-sqlite-vector `vector_distance_q` (int8) | quantized scan | +3.1s | 339 ms | 3.0 | 0.983 | 0.988 |
+| go-sqlite-vector `vector_distance` | exact scan | 2.6s | 385 ms | 2.6 | 0.999 | 1.000 |
+| go-sqlite-vector `vector_distance_q` (int8) | quantized scan | +3.3s | 347 ms | 2.9 | 0.983 | 0.988 |
+| go-sqlite-vector `vector_index` | exact chunk scan | 13.6s | 145 ms | 6.7 | 0.999 | 1.000 |
+| go-sqlite-vector `vector_index(int8)` | quantized chunk scan | 13.6s | 39 ms | 25.0 | 0.983 | 0.988 |
 | sqlite-vec 0.1.9 `vec_distance_l2` | exact scan | 1.4s | 306 ms | 3.3 | 0.999 | 1.000 |
 | sqlite-vec 0.1.9 `vec0` | exact scan | 4.4s | 143 ms | 6.9 | 0.999 | 1.000 |
 | FAISS 1.15 `IndexFlatL2` | exact, in memory | 0.0s | 7.9 ms | 125 | 0.999 | 1.000 |
@@ -216,13 +271,13 @@ Results on Apple M3 Max, macOS 15.7, Go 1.24.12. Build time is the time to inser
 
 Exact recall@10 is 0.999 rather than 1.000 because the ground truth contains tied distances. The int8 column stores 130 bytes per vector instead of 512.
 
-Scans read the whole table through SQLite's pager. Memory-mapping the database file avoids a `pread` system call per page and speeds up scans of large tables:
+Scans read the whole table through SQLite's pager. Memory-mapping the database file avoids a `pread` system call per page and speeds up scans of large tables. Set the size to at least the database file size:
 
 ```sql
-PRAGMA mmap_size = 1073741824; -- 1 GiB
+PRAGMA mmap_size = 4294967296; -- 4 GiB
 ```
 
-With this setting, SIFT1M float32 queries take 246 ms (p50) and int8 queries 286 ms. The comparison table above uses default settings for every SQLite implementation.
+With this setting, SIFT1M p50 latency drops to 248 ms for `vector_distance`, 287 ms for `vector_distance_q`, 64 ms for `vector_index`, and 17 ms for `vector_index(int8)`. The comparison table above uses default settings for every SQLite implementation.
 
 ## License
 

SPEC.md

@@ -106,6 +106,80 @@ Computes squared L2 distance between the dequantized forms of two quantized int8
 - **Output**: `REAL` (float64).
 - **Errors**: returns a SQL error if quantization was not configured. Returns a SQL error if either blob does not have the quantized format magic bytes (0x00, 0x01 prefix). Returns a SQL error if either blob's data length does not equal `dim` (total blob length = 2 + dim).
 
+## vector_index Virtual Table
+
+`Register` also registers a virtual table module named `vector_index`. It stores vectors in chunks and answers k-nearest neighbor queries by scanning chunks in Go instead of evaluating a SQL function per row.
+
+```sql
+CREATE VIRTUAL TABLE docs_vec USING vector_index();       -- float32 storage
+CREATE VIRTUAL TABLE docs_vec USING vector_index(float32); -- same
+CREATE VIRTUAL TABLE docs_vec USING vector_index(int8);    -- quantized storage
+CREATE VIRTUAL TABLE docs_vec USING vector_index(int8, chunk_size=4096);
+```
+
+Arguments are comma-separated and may appear in any order:
+
+- `float32` (default) or `int8`: storage type.
+- `chunk_size=N`: vectors per chunk, a positive integer. When omitted, `N = max(1, 1048576 / vectorBytes)`, where `vectorBytes` is `dim * 4` for float32 and `dim` for int8, so each chunk holds about 1 MiB of vector data.
+
+The storage type and chunk size are fixed when the table is created and read from the `name_info` shadow table on later connections.
+
+Columns:
+
+| Column | Type | Description |
+|---|---|---|
+| `rowid` | INTEGER | Row identifier. Assigned automatically when omitted on insert. |
+| `embedding` | BLOB | The vector. Written as a float32 blob of `dim * 4` bytes. Read back as a float32 blob, or a quantized blob for int8 tables. |
+| `distance` | REAL, hidden | Squared L2 distance to the `MATCH` query. NULL outside a search. |
+| `k` | INTEGER, hidden | Number of nearest neighbors to return. |
+
+### Search
+
+A search constrains `embedding MATCH` with a float32 query blob and gives k either as a `k` constraint or as `LIMIT`:
+
+```sql
+SELECT rowid, distance FROM docs_vec WHERE embedding MATCH ?1 AND k = 10;
+
+SELECT rowid, distance FROM docs_vec WHERE embedding MATCH ?1 ORDER BY distance LIMIT 10;
+```
+
+- Results are ordered by ascending distance, then rowid.
+- With `LIMIT n OFFSET m`, the search returns the `n + m` nearest rows and SQLite applies the offset.
+- SQLite passes `LIMIT` to the virtual table only when the query reads that table alone. Queries that join, aggregate, or group must use `k`, or put the search in a subquery with its own `LIMIT`.
+- For int8 tables, the query vector is quantized with the configured range and distances equal `vector_distance_q`.
+- The chunk scan runs on `runtime.GOMAXPROCS(0)` goroutines.
+
+### Writes
+
+`INSERT`, `UPDATE`, and `DELETE` are supported. Inserts and updates take a float32 blob; int8 tables quantize it on write, clamping values outside the configured range.
+
+### Storage
+
+Each `vector_index` table `name` uses four shadow tables in the same database:
+
+- `name_info`: dimension, storage type, quantization range, and chunk size at creation.
+- `name_chunks`: chunk id and number of occupied slots.
+- `name_data`: per chunk, a blob of little-endian int64 rowids and a blob of vectors, each with capacity for the table's chunk size. int8 chunks store raw int8 values without the quantized blob header.
+- `name_rowids`: rowid to chunk and slot.
+
+Deleting a vector moves the chunk's last vector into the freed slot, so occupied slots in every chunk are contiguous. Shadow tables are ordinary tables, so writes follow SQLite transactions. `ALTER TABLE ... RENAME` and `DROP TABLE` rename and drop the shadow tables.
+
+### Errors
+
+| Condition | Error message format |
+|---|---|
+| Unknown argument | `"vector_index: unknown argument %q, expected float32, int8, or chunk_size=N"` |
+| Invalid chunk size | `"vector_index: chunk_size must be a positive integer, got %q"` |
+| int8 without quantization | `"vector_index: int8 storage requires Register with WithQuantRange"` |
+| Dimension differs from creation | `"vector_index: table %s has dimension %d, Register was called with %d"` |
+| Storage type differs from creation | `"vector_index: table %s stores %s, declaration says %s"` |
+| Quantization range differs from creation | `"vector_index: table %s was created with quantization range [%g, %g], Register was called with [%g, %g]"` |
+| Embedding or query not a blob | `"vector_index: %s must be a float32 blob, got %v"` |
+| Embedding or query size mismatch | `"vector_index: %s: expected %d bytes (dim=%d), got %d"` |
+| Duplicate rowid | `"vector_index: UNIQUE constraint failed: %s.rowid (%d)"` |
+| Search without k or LIMIT | `"vector_index: search requires k = ? or LIMIT"` |
+| Negative k | `"vector_index: k must be non-negative, got %d"` |
+
 ## Blob Formats
 
 ### Float32 Blob
@@ -242,6 +316,7 @@ Table-driven integration tests executed against real SQLite connections. Each te
 - **vector_distance_q**: correct distance after dequantization, format validation, not-configured error, NULL input.
 - **Round-trip**: encode -> store -> retrieve -> distance pipeline.
 - **Float32ToBlob / BlobToFloat32**: Go-level encode/decode correctness.
+- **vector_index**: search results match `vector_distance` / `vector_distance_q` brute force for `k`, `LIMIT`, `OFFSET`, and joins; results stay correct after deletes, updates, rowid changes, and rollbacks; stored embeddings read back unchanged; schema checks on reconnect; rename and drop.
 
 ## Benchmarks
 
@@ -254,6 +329,5 @@ Go benchmarks (`testing.B`) for the core operations at dimensions 384, 768, and
 
 ## Design Notes
 
-- **Extensibility**: the package is designed so that a virtual table module (via `sqlite.SetModule` / `SetModule`) could be added in the future without breaking the existing scalar function API.
 - **No CGo**: all vector math is implemented in pure Go using `encoding/binary` and `math` from the standard library.
-- **Nearest neighbor scan**: search is a brute-force scan over all rows. This is appropriate for SQLite-scale datasets (thousands to low millions of vectors). For larger datasets, users should consider a dedicated vector database.
+- **Nearest neighbor scan**: search is a brute-force scan over all rows, through either the scalar functions or `vector_index`. This is appropriate for SQLite-scale datasets (thousands to low millions of vectors). For larger datasets, users should consider a dedicated vector database.

go.mod

@@ -4,7 +4,7 @@ go 1.23.0
 
 toolchain go1.24.12
 
-require zombiezen.com/go/sqlite v1.4.2
+require zombiezen.com/go/sqlite v1.4.3-0.20260620165750-ca2041ed3dd2
 
 require (
 	github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

@@ -47,5 +47,5 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
 modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
 modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-zombiezen.com/go/sqlite v1.4.2 h1:KZXLrBuJ7tKNEm+VJcApLMeQbhmAUOKA5VWS93DfFRo=
-zombiezen.com/go/sqlite v1.4.2/go.mod h1:5Kd4taTAD4MkBzT25mQ9uaAlLjyR0rFhsR6iINO70jc=
+zombiezen.com/go/sqlite v1.4.3-0.20260620165750-ca2041ed3dd2 h1:gAUUjwioO8gmtw4a/5p8MmzYNWhLOGw32Sz1yXi4KnM=
+zombiezen.com/go/sqlite v1.4.3-0.20260620165750-ca2041ed3dd2/go.mod h1:5Kd4taTAD4MkBzT25mQ9uaAlLjyR0rFhsR6iINO70jc=

sift_test.go

@@ -72,6 +72,16 @@ func TestSIFT1M(t *testing.T) {
 	exec(t, conn, "UPDATE sift SET embedding_q = vector_quantize(embedding)")
 	quantizeTime := time.Since(start)
 
+	exec(t, conn, "CREATE VIRTUAL TABLE sift_index USING vector_index(float32)")
+	start = time.Now()
+	exec(t, conn, "INSERT INTO sift_index (rowid, embedding) SELECT id, embedding FROM sift")
+	indexTime := time.Since(start)
+
+	exec(t, conn, "CREATE VIRTUAL TABLE sift_index_q USING vector_index(int8)")
+	start = time.Now()
+	exec(t, conn, "INSERT INTO sift_index_q (rowid, embedding) SELECT id, embedding FROM sift")
+	indexQTime := time.Since(start)
+
 	fi, err := os.Stat(dbPath)
 	if err != nil {
 		t.Fatal(err)
@@ -80,8 +90,10 @@ func TestSIFT1M(t *testing.T) {
 	t.Logf("SIFT1M: %d base vectors, %d queries, dim=%d, quant range [%g, %g]", len(base), nq, dim, lo, hi)
 	t.Logf("insert float32:       %v", insertTime.Round(time.Millisecond))
 	t.Logf("quantize column:      %v", quantizeTime.Round(time.Millisecond))
-	t.Logf("database size:        %.1f MiB (both columns)", float64(fi.Size())/(1<<20))
-	t.Logf("%-10s %10s %10s %10s %8s %8s %8s", "method", "p50", "p99", "QPS", "R@1", "R@10", "R@100")
+	t.Logf("insert index float32: %v", indexTime.Round(time.Millisecond))
+	t.Logf("insert index int8:    %v", indexQTime.Round(time.Millisecond))
+	t.Logf("database size:        %.1f MiB (all tables)", float64(fi.Size())/(1<<20))
+	t.Logf("%-14s %10s %10s %10s %8s %8s %8s", "method", "p50", "p99", "QPS", "R@1", "R@10", "R@100")
 
 	for _, m := range []struct {
 		name  string
@@ -90,6 +102,8 @@ func TestSIFT1M(t *testing.T) {
 	}{
 		{"float32", "SELECT id FROM sift ORDER BY vector_distance(embedding, ?1) LIMIT 100", Float32ToBlob},
 		{"int8", "SELECT id FROM sift ORDER BY vector_distance_q(embedding_q, ?1) LIMIT 100", func(v []float32) []byte { return quantize(v, lo, hi) }},
+		{"index float32", "SELECT rowid FROM sift_index WHERE embedding MATCH ?1 AND k = 100", Float32ToBlob},
+		{"index int8", "SELECT rowid FROM sift_index_q WHERE embedding MATCH ?1 AND k = 100", Float32ToBlob},
 	} {
 		lat := make([]time.Duration, nq)
 		var r1, r10, r100 float64
@@ -120,7 +134,7 @@ func TestSIFT1M(t *testing.T) {
 		}
 		slices.Sort(lat)
 		n := float64(nq)
-		t.Logf("%-10s %10v %10v %10.2f %8.4f %8.4f %8.4f", m.name,
+		t.Logf("%-14s %10v %10v %10.2f %8.4f %8.4f %8.4f", m.name,
 			lat[nq/2].Round(time.Millisecond), lat[nq*99/100].Round(time.Millisecond),
 			n/total.Seconds(), r1/n, r10/n, r100/n)
 	}

vector.go

@@ -202,6 +202,14 @@ func Register(conn *sqlite.Conn, dim int, opts ...Option) error {
 		return err
 	}
 
+	err = conn.SetModule("vector_index", &sqlite.Module{
+		Create:  indexConnect(cfg, true),
+		Connect: indexConnect(cfg, false),
+	})
+	if err != nil {
+		return err
+	}
+
 	err = conn.SetModule("vector_chunk", &sqlite.Module{
 		Connect: func(c *sqlite.Conn, opts *sqlite.VTableConnectOptions) (sqlite.VTable, *sqlite.VTableConfig, error) {
 			return &chunkVTable{chunker: cfg.chunker}, &sqlite.VTableConfig{
@@ -269,14 +277,19 @@ func f32At(b []byte, i int) float64 {
 // Dequantization is affine, so each component difference is
 // (qa - qb) * (max - min) / 255.
 func l2SquaredQuantized(a, b []byte, min, max float32) float64 {
-	a, b = a[2:], b[2:len(a)]
+	scale := float64(max-min) / 255
+	return float64(l2SquaredInt8(a[2:], b[2:])) * scale * scale
+}
+
+// l2SquaredInt8 sums squared differences of raw int8 values of equal length.
+func l2SquaredInt8(a, b []byte) int64 {
+	b = b[:len(a)]
 	var sum int64
 	for i := range a {
 		d := int64(int8(a[i])) - int64(int8(b[i]))
 		sum += d * d
 	}
-	scale := float64(max-min) / 255
-	return float64(sum) * scale * scale
+	return sum
 }
 
 func isQuantizedBlob(b []byte) bool {