From a675b0d07064df1e41054c7c546755fa70689e6f Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Tue, 25 Aug 2026 09:39:34 +0200 Subject: [PATCH 1/3] docs: pivot the benchmark table around hardware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The table was organised by mode inside a per-machine section, which answers "which mode should I pick" — a question whose answer does not vary by CPU. The comparison people actually want from a benchmark table is across hardware, and that one was buried. The section now has two tables. The hardware table carries INT8 only, two rows per machine, each row self-contained: hardware, vectors, index configuration, peak memory during the scan, latency, throughput, recall. Someone comparing two rows does not have to carry the prose above the table in their head, which matters when the rows being compared are far apart. Everything that does not vary by machine moved into the parameters above it: the dataset and its seed, the 740 MB index, and the exact FLOAT32 scan recall is measured against. Adding a machine is two rows. Those two rows are the two ways the same index gets deployed — entirely resident after vector_quantize_preload(), or streamed through a bounded buffer with max_memory=30MB, which is the default and what a device with 740 MB of index and less RAM than that actually does. The streaming configuration now passes max_memory explicitly rather than relying on the default staying 30 MB. Max memory is measured rather than the parameter echoed back: the benchmark brackets vector_quantize_preload() with sqlite3_memory_used() for the resident case and takes the high-water mark across the query loop for the streamed one. On 1M x 768 that reads 740 MB against 30 MB, which is worth having as a measurement — it shows max_memory bounds what the scan holds, not just the size of the chunks written at quantization time. The two configurations differ by 25x in memory for 1.5x in latency with recall untouched, since both read the same index and only residency changes. `make benchmark HARDWARE="Apple M5 Pro - NEON"` prints those two rows ready to paste, in the table's column order and with thousands separators, so a contributor reformats nothing. The label is CPU and backend only: core counts and RAM were noise in a column meant to be scanned down, and the backend is the part that can surprise you. Recall is repeated on every row even though it depends on the data rather than the machine. A row that disagrees with the others is a sign that machine selected a different SIMD backend than it should have, which is exactly the class of bug that had every x86 build shipping scalar code. Mode selection keeps its own table, since the three findings worth knowing — INT8 over UINT8 for cosine, 1BIT as a pre-filter, TurboQuant trading size rather than speed — are properties of the data and the algorithms, not of the CPU. Co-Authored-By: Claude Opus 5 --- Makefile | 5 ++- README.md | 107 +++++++++++++++++++++++++++-------------------- test/benchmark.c | 62 ++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index 6f3055a..b9a4b4f 100644 --- a/Makefile +++ b/Makefile @@ -188,11 +188,14 @@ unittest-simd: $(BUILD_DIR)/backend $(BUILD_DIR)/test_vector_simd # make benchmark k=20 over 1M vectors of dim 768 # make benchmark NVECS=100000 DIM=384 K=10 smaller, for a quick look # make benchmark DISTANCE=l2 a different metric +# make benchmark HARDWARE="M5 Pro - NEON" label the row it prints for the README NVECS ?= 1000000 DIM ?= 768 K ?= 20 NQUERIES ?= 20 DISTANCE ?= cosine +# names the row this run contributes to the README hardware table +HARDWARE ?= - backend BENCH_OBJ = $(patsubst %.c, $(BUILD_DIR)/bm-%.o, $(notdir $(SRC_FILES))) $(BUILD_DIR)/bm-sqlite3.o @@ -203,7 +206,7 @@ $(BUILD_DIR)/bm-%.o: %.c $(CC) $(CFLAGS) $(ISA_CFLAGS) -DSQLITE_CORE -O3 -c $< -o $@ $(BUILD_DIR)/benchmark: test/benchmark.c $(BENCH_OBJ) - $(CC) $(CFLAGS) -DSQLITE_CORE -O3 -DNVECS=$(NVECS) -DDIM=$(DIM) -DK=$(K) -DNQUERIES=$(NQUERIES) -DDISTANCE='"$(DISTANCE)"' $< $(BENCH_OBJ) -o $@ -lm -lpthread + $(CC) $(CFLAGS) -DSQLITE_CORE -O3 -DNVECS=$(NVECS) -DDIM=$(DIM) -DK=$(K) -DNQUERIES=$(NQUERIES) -DDISTANCE='"$(DISTANCE)"' -DHARDWARE='"$(HARDWARE)"' $< $(BENCH_OBJ) -o $@ -lm -lpthread benchmark: $(BUILD_DIR)/benchmark ./$(BUILD_DIR)/benchmark diff --git a/README.md b/README.md index e662f9e..8316d69 100644 --- a/README.md +++ b/README.md @@ -147,67 +147,84 @@ SELECT e.id, v.distance FROM images AS e ## Benchmark -Every number below comes from one command, so you can reproduce it and compare machines: +One command, so results from different machines are comparable: ```bash -make benchmark +make benchmark HARDWARE="Apple M5 Pro - NEON" ``` -That builds `test/benchmark.c` at `-O3` with the same per-translation-unit SIMD flags the -shipped extension uses, then searches **k=20 over 1,000,000 vectors of dimension 768** -with cosine distance, 20 queries, reporting the best. Recall is the overlap with the exact -full-precision top-20. Override any of it: +It builds `test/benchmark.c` at `-O3` with the same per-translation-unit SIMD flags the +shipped extension uses, then runs **k=20 over 1,000,000 vectors of dimension 768** with +cosine distance, 20 queries, reporting the best. It prints the two rows below ready to +paste. Override anything: ```bash make benchmark NVECS=100000 DIM=384 K=10 DISTANCE=l2 ``` -### Apple M5 Pro (6P+12E, 64 GB, macOS 26.6.2) — NEON backend +Common to every row: vectors are **uniform random** with a fixed seed, so two machines +measure the same data; the `INT8` index is **740 MB** on disk against 2930 MB of raw +`FLOAT32`; recall is the overlap with the exact `FLOAT32` scan, which is the baseline +everything is compared against and is 100% by definition. On the reference machine that +exact scan takes **148.3 ms/query**. -| Mode | Index | ms/query | Mvec/s | Recall@20 | -| --- | ---: | ---: | ---: | ---: | -| `FLOAT32` exact | 2930 MB | 147.8 | 6.8 | 100.0% | -| `UINT8` | 740 MB | 55.4 | 18.0 | 33.8% | -| `UINT8` preloaded | 740 MB | 37.2 | 26.9 | 33.8% | -| `INT8` | 740 MB | 56.4 | 17.7 | 99.5% | -| **`INT8` preloaded** | **740 MB** | **37.7** | **26.5** | **99.5%** | -| `1BIT` | 99 MB | 5.3 | 187.5 | 10.0% | -| `1BIT` preloaded | 99 MB | 2.7 | 377.6 | 10.0% | -| `TURBO2` | 195 MB | 53.0 | 18.9 | 45.2% | -| `TURBO2` preloaded | 195 MB | 48.2 | 20.7 | 45.2% | -| `TURBO4` | 378 MB | 160.4 | 6.2 | 81.8% | -| `TURBO4` preloaded | 378 MB | 151.8 | 6.6 | 81.8% | +The two rows per machine are the two ways the same index gets deployed. *Preloaded* holds +it entirely in RAM after `vector_quantize_preload()`. *Streamed* walks it through a bounded +buffer set by `max_memory=30MB`, which is the default and what a device with 740 MB of +index and less RAM than that actually does. The **Max memory** column is measured, not the +parameter echoed back: it is what the extension had allocated at the peak of the scan. -*Contributions from other CPUs welcome — run the command above and open a PR adding a -section.* +### Hardware -### Reading the table +| Hardware | Vectors | Index | Max memory | ms/query | Mvec/s | Recall@20 | +| --- | ---: | --- | ---: | ---: | ---: | ---: | +| Apple M5 Pro - NEON | 1,000,000 | `INT8` preloaded | 740 MB | 37.3 | 26.8 | 99.5% | +| Apple M5 Pro - NEON | 1,000,000 | `INT8` streamed | 30 MB | 56.2 | 17.8 | 99.5% | -**The data is uniform random**, which is the worst case for every quantizer: real -embeddings have structure that quantization exploits, so recall on your own vectors will -be higher, often much higher. Treat the recall column as a floor and a way to rank the -modes against each other, not as a prediction for your dataset. +*Results from other CPUs welcome — run the command above and open a PR adding your two +rows.* -Three things are worth knowing before you pick a mode. +The trade is 25x less memory for 1.5x the latency, and recall is untouched because both +rows read the same index — only how much of it is resident differs. -**For cosine, use `INT8`, not `UINT8`.** They cost exactly the same and store exactly the -same number of bytes, but `UINT8` recall collapses to 33.8% while `INT8` holds 99.5%. +Recall is repeated per row on purpose: it depends on the data, not the hardware, so a row +that disagrees with the others is a sign that machine selected a different SIMD backend +than it should have. + +### Choosing a mode + +`INT8` is in the table because it is the mode to reach for first. The others, measured on +the same machine and data: + +| Mode | Index | ms/query | Recall@20 | +| --- | ---: | ---: | ---: | +| `FLOAT32` exact | 2930 MB | 148.3 | 100.0% | +| `UINT8` preloaded | 740 MB | 37.2 | 33.8% | +| `INT8` preloaded | 740 MB | 37.3 | 99.5% | +| `1BIT` preloaded | 99 MB | 2.4 | 10.0% | +| `TURBO2` preloaded | 195 MB | 47.1 | 45.2% | +| `TURBO4` preloaded | 378 MB | 151.3 | 81.8% | + +**The data is uniform random**, the worst case for every quantizer: real embeddings have +structure quantization exploits, so recall on your own vectors will be higher, often much +higher. Read that column as a floor and a way to rank the modes, not as a prediction. + +Three things are worth knowing before choosing. + +**For cosine, use `INT8`, not `UINT8`.** Same size, same speed, 33.8% recall against 99.5%. Unsigned quantization subtracts the dataset minimum before scaling, and cosine measures -angle, which that shift destroys. `UINT8` is the right choice for L2, where a common -translation cancels out. If you do not set `qtype`, the extension picks `UINT8` for -non-negative data and `INT8` otherwise — which is the correct call for L2 and the wrong -one for cosine, so set it explicitly when you use cosine. - -**`1BIT` is a filter, not an answer.** 377 Mvec/s and 30x less memory, at 10% recall on -this data. It earns its place as a first pass whose survivors you re-rank at full -precision, not as the final ranking. - -**TurboQuant trades speed for size, not for speed.** `TURBO4` here is *slower* than the -exact scan (160 ms against 148 ms) while using 8x less memory and returning 81.8% recall. -The lookup-table scan is one table gather per row, and at dimension 768 that is 384 -gathers into a 393 KB table for every vector — already about one lookup per cycle, so -there is no headroom left in the current storage layout. Choose TurboQuant when the -memory budget is what binds; choose `INT8` when throughput is. +angle, which that shift destroys. `UINT8` is right for L2, where a common translation +cancels. If you omit `qtype` the extension picks `UINT8` for non-negative data — correct +for L2, wrong for cosine — so set it explicitly when you use cosine. + +**`1BIT` is a filter, not an answer.** 409 Mvec/s and 30x less memory, at 10% recall here. +It earns its place as a first pass whose survivors you re-rank at full precision. + +**TurboQuant trades size, not speed.** `TURBO4` is *slower* than the exact scan while using +8x less memory. Its lookup scan is one table gather per row — at dimension 768 that is 384 +gathers into a 384 KB table per vector, already about one lookup per cycle, so the current +storage layout has no headroom left ([#57](https://github.com/sqliteai/sqlite-vector/issues/57)). +Choose TurboQuant when memory is what binds; choose `INT8` when throughput is. ## TurboQuant Benchmark and Recall diff --git a/test/benchmark.c b/test/benchmark.c index 54e3c17..1c3053b 100644 --- a/test/benchmark.c +++ b/test/benchmark.c @@ -39,6 +39,9 @@ extern int sqlite3_vector_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_ #ifndef DISTANCE #define DISTANCE "cosine" #endif +#ifndef HARDWARE +#define HARDWARE " — backend" +#endif // xorshift64*: the data must be identical from run to run and from machine to machine, // and rand() is neither fast enough nor portable enough for that @@ -116,6 +119,19 @@ static sqlite3_int64 index_bytes (sqlite3 *db) { return bytes; } +// 1000000 is hard to read in a table cell +static const char *with_separators (long long n, char *buf, size_t cap) { + char digits[32]; + snprintf(digits, sizeof(digits), "%lld", n); + size_t len = strlen(digits), out = 0; + for (size_t i = 0; i < len && out + 2 < cap; ++i) { + if (i > 0 && ((len - i) % 3) == 0) buf[out++] = ','; + buf[out++] = digits[i]; + } + buf[out] = 0; + return buf; +} + static void report (const char *label, sqlite3_int64 bytes, double seconds, double recall) { printf("| %-24s | %8.1f | %9.2f | %9.1f | %6.1f |\n", label, @@ -172,14 +188,20 @@ int main (void) { double exact_time = measure(db, "vector_full_scan", NULL, NULL); report("FLOAT32 exact", 0, exact_time, 100.0); + // max_memory bounds the chunk the scan streams through when the index is not + // preloaded, and is the whole point of that configuration: the index here is 740 MB, + // the scan walks it 30 MB at a time struct { const char *opts; const char *label; } modes[] = { - { "qtype=UINT8", "UINT8" }, - { "qtype=INT8", "INT8" }, - { "qtype=1BIT", "1BIT" }, - { "qtype=TURBO,qbits=2", "TURBO2" }, - { "qtype=TURBO,qbits=4", "TURBO4" }, + { "qtype=UINT8,max_memory=30MB", "UINT8" }, + { "qtype=INT8,max_memory=30MB", "INT8" }, + { "qtype=1BIT,max_memory=30MB", "1BIT" }, + { "qtype=TURBO,qbits=2,max_memory=30MB", "TURBO2" }, + { "qtype=TURBO,qbits=4,max_memory=30MB", "TURBO4" }, }; + double int8_stream_ms = 0, int8_pre_ms = 0, int8_recall = 0; + sqlite3_int64 int8_bytes = 0, int8_stream_mem = 0, int8_pre_mem = 0; + for (unsigned m = 0; m < sizeof(modes) / sizeof(modes[0]); ++m) { char sql[160], label[64]; fprintf(stderr, "quantizing %s...\n", modes[m].label); @@ -188,18 +210,46 @@ int main (void) { sqlite3_int64 bytes = index_bytes(db); double recall = 0.0; + sqlite3_int64 before = sqlite3_memory_used(); + sqlite3_memory_highwater(1); double t = measure(db, "vector_quantize_scan", exact, &recall); - snprintf(label, sizeof(label), "%s", modes[m].label); + sqlite3_int64 stream_peak = sqlite3_memory_highwater(0) - before; + snprintf(label, sizeof(label), "%s (30 MB)", modes[m].label); report(label, bytes, t, recall); + if (strcmp(modes[m].label, "INT8") == 0) { int8_stream_ms = t; int8_stream_mem = stream_peak; } + before = sqlite3_memory_used(); run_sql(db, "SELECT vector_quantize_preload('t','v');"); + sqlite3_int64 preload_held = sqlite3_memory_used() - before; t = measure(db, "vector_quantize_scan", exact, &recall); snprintf(label, sizeof(label), "%s preloaded", modes[m].label); report(label, bytes, t, recall); + if (strcmp(modes[m].label, "INT8") == 0) { + int8_bytes = bytes; + int8_recall = recall; + int8_pre_ms = t; + int8_pre_mem = preload_held; + } + run_sql(db, "SELECT vector_quantize_cleanup('t','v');"); } + // The README table compares hardware, so it carries the two configurations that + // differ by deployment rather than by accuracy: the whole index in RAM, and the same + // index streamed in 30 MB. Everything else about INT8 is a property of the data. + char nbuf[32]; + with_separators(NVECS, nbuf, sizeof(nbuf)); + printf("\n\nPaste these two rows into the hardware table in README.md:\n\n"); + printf("| %s | %s | `INT8` preloaded | %.0f MB | %.1f | %.1f | %.1f%% |\n", + HARDWARE, nbuf, (double)int8_pre_mem / (1024.0 * 1024.0), + int8_pre_ms * 1000.0, NVECS / int8_pre_ms / 1e6, int8_recall); + printf("| %s | %s | `INT8` streamed | %.0f MB | %.1f | %.1f | %.1f%% |\n", + HARDWARE, nbuf, (double)int8_stream_mem / (1024.0 * 1024.0), + int8_stream_ms * 1000.0, NVECS / int8_stream_ms / 1e6, int8_recall); + printf("\nreference for this machine: FLOAT32 exact %.1f ms/query, index on disk %.0f MB\n", + exact_time * 1000.0, (double)int8_bytes / (1024.0 * 1024.0)); + sqlite3_close(db); return 0; } From 7391418c3498ad1d68b9107035316a0ccd0f1a7e Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Tue, 25 Aug 2026 09:48:15 +0200 Subject: [PATCH 2/3] docs: benchmark on a file-backed database, never in memory An in-memory database puts the whole index in the process no matter how the scan is configured, which makes the memory column meaningless: the streamed row's whole point is that the index is not resident, and :memory: made that untrue by construction. The benchmark now opens a file and deletes it afterwards. It changes the numbers, and it changes one conclusion I had written. FLOAT32 exact 148 -> 486 ms reads 3 GB per query from the file INT8 streamed 56 -> 116 ms reads 740 MB through SQLite INT8 preloaded 37.3 -> 37.6 ms unchanged That last line is the interesting one: preloading makes the scan almost independent of where the database lives, because after the one-time load it reads the extension's own buffer and never goes back to SQLite. The corrected conclusion is about TurboQuant. Measured in memory, TURBO4 came out slower than the exact scan, and the README said so. On a file-backed database it is 3.2x faster, because the exact scan is reading 3 GB per query while TURBO4 reads 378 MB. TurboQuant does beat brute force; what it does not beat is INT8, which is 4x faster again at twice the size. That is the honest framing and it is now what both the benchmark section and the older TurboQuant section say - the latter had a note contextualising its 15x/38x claims as an artifact of being file-backed, which is no longer the distinction that explains the gap. The memory column is also honest about what it does not cover: timings are best-of-20, so the file is in the operating system's page cache, which lives outside the process and does not appear in the column. A device where the index genuinely does not fit adds storage reads on top. Co-Authored-By: Claude Opus 5 --- README.md | 95 +++++++++++++++++++++++++++--------------------- test/benchmark.c | 24 ++++++++++-- 2 files changed, 74 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 8316d69..18f1370 100644 --- a/README.md +++ b/README.md @@ -162,48 +162,58 @@ paste. Override anything: make benchmark NVECS=100000 DIM=384 K=10 DISTANCE=l2 ``` -Common to every row: vectors are **uniform random** with a fixed seed, so two machines -measure the same data; the `INT8` index is **740 MB** on disk against 2930 MB of raw -`FLOAT32`; recall is the overlap with the exact `FLOAT32` scan, which is the baseline -everything is compared against and is 100% by definition. On the reference machine that -exact scan takes **148.3 ms/query**. +Common to every row: the database is **a file, never `:memory:`** — an in-memory database +puts the whole index in the process no matter how it is configured, which makes any memory +figure meaningless. Vectors are uniform random with a fixed seed, so two machines measure +the same data. The `INT8` index is **740 MB** on disk against 2930 MB of raw `FLOAT32`. +Recall is the overlap with the exact `FLOAT32` scan, the baseline everything is compared +against, 100% by definition; on the reference machine that scan takes **486 ms/query**, +because it reads 3 GB per query. The two rows per machine are the two ways the same index gets deployed. *Preloaded* holds -it entirely in RAM after `vector_quantize_preload()`. *Streamed* walks it through a bounded -buffer set by `max_memory=30MB`, which is the default and what a device with 740 MB of -index and less RAM than that actually does. The **Max memory** column is measured, not the -parameter echoed back: it is what the extension had allocated at the peak of the scan. +it in the process after `vector_quantize_preload()`. *Streamed* walks it through a bounded +buffer set by `max_memory=30MB`, the default, and what a device with less RAM than the +index actually does. **Max memory** is measured, not the parameter echoed back: it is the +peak the extension and SQLite had allocated during the scan. ### Hardware | Hardware | Vectors | Index | Max memory | ms/query | Mvec/s | Recall@20 | | --- | ---: | --- | ---: | ---: | ---: | ---: | -| Apple M5 Pro - NEON | 1,000,000 | `INT8` preloaded | 740 MB | 37.3 | 26.8 | 99.5% | -| Apple M5 Pro - NEON | 1,000,000 | `INT8` streamed | 30 MB | 56.2 | 17.8 | 99.5% | +| Apple M5 Pro - NEON | 1,000,000 | `INT8` preloaded | 740 MB | 37.6 | 26.6 | 99.5% | +| Apple M5 Pro - NEON | 1,000,000 | `INT8` streamed | 30 MB | 115.5 | 8.7 | 99.5% | *Results from other CPUs welcome — run the command above and open a PR adding your two rows.* -The trade is 25x less memory for 1.5x the latency, and recall is untouched because both -rows read the same index — only how much of it is resident differs. +The trade is **25x less memory for 3x the latency**, with recall untouched: both rows read +the same index, only how much of it is resident differs. -Recall is repeated per row on purpose: it depends on the data, not the hardware, so a row -that disagrees with the others is a sign that machine selected a different SIMD backend -than it should have. +Two things the timings do not show. They are best-of-20, so the file is in the operating +system's page cache by then — that cache lives outside the process and is evicted under +pressure, so it is not in the memory column, but it is why the streamed row is not paying +for storage reads. On a device where the index genuinely does not fit in RAM, add +`index size / storage bandwidth` to the streamed number. And the preloaded row is almost +unaffected by where the database lives, because after the one-time preload the scan reads +the extension's own buffer and never goes back to SQLite. + +Recall is repeated on every row on purpose: it depends on the data, not the hardware, so a +row that disagrees with the others is a sign that machine selected a different SIMD +backend than it should have. ### Choosing a mode -`INT8` is in the table because it is the mode to reach for first. The others, measured on -the same machine and data: +`INT8` is in the table above because it is the mode to reach for first. The others, same +machine, same data, all preloaded: -| Mode | Index | ms/query | Recall@20 | -| --- | ---: | ---: | ---: | -| `FLOAT32` exact | 2930 MB | 148.3 | 100.0% | -| `UINT8` preloaded | 740 MB | 37.2 | 33.8% | -| `INT8` preloaded | 740 MB | 37.3 | 99.5% | -| `1BIT` preloaded | 99 MB | 2.4 | 10.0% | -| `TURBO2` preloaded | 195 MB | 47.1 | 45.2% | -| `TURBO4` preloaded | 378 MB | 151.3 | 81.8% | +| Mode | Index | ms/query | vs exact | Recall@20 | +| --- | ---: | ---: | ---: | ---: | +| `FLOAT32` exact | 2930 MB | 486.5 | 1.0x | 100.0% | +| `UINT8` | 740 MB | 37.5 | 13.0x | 33.8% | +| `INT8` | 740 MB | 37.6 | 13.0x | 99.5% | +| `1BIT` | 99 MB | 2.7 | 183x | 10.0% | +| `TURBO2` | 195 MB | 47.8 | 10.2x | 45.2% | +| `TURBO4` | 378 MB | 150.9 | 3.2x | 81.8% | **The data is uniform random**, the worst case for every quantizer: real embeddings have structure quantization exploits, so recall on your own vectors will be higher, often much @@ -217,14 +227,17 @@ angle, which that shift destroys. `UINT8` is right for L2, where a common transl cancels. If you omit `qtype` the extension picks `UINT8` for non-negative data — correct for L2, wrong for cosine — so set it explicitly when you use cosine. -**`1BIT` is a filter, not an answer.** 409 Mvec/s and 30x less memory, at 10% recall here. -It earns its place as a first pass whose survivors you re-rank at full precision. +**`1BIT` is a filter, not an answer.** 183x faster than exact and 30x smaller, at 10% +recall here. It earns its place as a first pass whose survivors you re-rank at full +precision. -**TurboQuant trades size, not speed.** `TURBO4` is *slower* than the exact scan while using -8x less memory. Its lookup scan is one table gather per row — at dimension 768 that is 384 -gathers into a 384 KB table per vector, already about one lookup per cycle, so the current -storage layout has no headroom left ([#57](https://github.com/sqliteai/sqlite-vector/issues/57)). -Choose TurboQuant when memory is what binds; choose `INT8` when throughput is. +**TurboQuant buys memory against `INT8`, not speed.** `TURBO4` is 3.2x faster than the +exact scan, so it is a real win over brute force — but `INT8` is 4x faster again at twice +the size, and `TURBO2` is both smaller and faster than `TURBO4` if 45% recall is enough. +TurboQuant's lookup scan is one table gather per row: at dimension 768 that is 384 gathers +into a 384 KB table per vector, already about one lookup per cycle, so its current storage +layout has no headroom left ([#57](https://github.com/sqliteai/sqlite-vector/issues/57)). +Reach for it when the memory budget is what binds. ## TurboQuant Benchmark and Recall @@ -238,15 +251,13 @@ SELECT vector_quantize('images', 'embedding', 'qtype=TURBO,qbits=4'); SELECT vector_quantize('images', 'embedding', 'qtype=TURBO2'); ``` -An earlier synthetic benchmark reported speedups of 15x for 4-bit and 38x for 2-bit -against `vector_full_scan()`. Those numbers were measured with a **file-backed** database, -where the full scan reads 3 GB of raw vectors off disk and the comparison is dominated by -I/O rather than by arithmetic — and before the distance kernels were rewritten, which made -the full-precision scan itself substantially faster. Against an in-memory baseline on -current code the picture is different: see [Benchmark](#benchmark) below, where `TURBO4` -is marginally slower than the exact scan and its argument is memory, not speed. Both -measurements are real; they answer different questions. If your working set does not fit -in RAM, the file-backed comparison is the one that describes your deployment. +An earlier synthetic benchmark on this dataset reported speedups of 15x for 4-bit and 38x +for 2-bit against `vector_full_scan()`, with DOT distance and k=10. The [Benchmark](#benchmark) +section above measures the same shape with cosine and k=20 and lands lower — 3.2x for +4-bit, 10.2x for 2-bit — mostly because the distance kernels have since been rewritten, +which made the full-precision baseline it is compared against substantially faster. The +direction is the same: TurboQuant beats brute force, and `INT8` beats TurboQuant on speed +while costing twice the memory. For comparison, the raw `FLOAT32` vectors alone are about **3.07 GB** for 1M x 768 before SQLite row/page overhead. TurboQuant 4-bit reduces the scan representation to about **13%** of that raw vector payload, TurboQuant 3-bit to about **10%**, and TurboQuant 2-bit to about **7%**. Actual resident memory depends on whether the database is in-memory or file-backed, SQLite cache settings, preloading, page cache behavior, and the host allocator. diff --git a/test/benchmark.c b/test/benchmark.c index 1c3053b..4f7616c 100644 --- a/test/benchmark.c +++ b/test/benchmark.c @@ -9,8 +9,15 @@ // Defaults to k=20 over 1,000,000 vectors of dimension 768. Override at build time: // make benchmark NVECS=100000 DIM=384 K=10 NQUERIES=20 // -// The database is in memory, so "on disk" below means the index is read back through -// SQLite rather than from the extension's preloaded buffer - not filesystem I/O. +// The database is a file, never :memory:. An in-memory database puts the whole index in +// the process regardless of configuration, which makes the memory column meaningless - +// the point of the streamed row is that the index is not resident. +// +// Timings are best-of-N, so they are the warm case: after the first query the file is in +// the operating system's page cache. That cache is outside the process and is evicted +// under pressure, so it does not appear in the memory column - but it is why a streamed +// scan is not paying for storage reads here. A cold read of the whole index would add +// index_size / storage_bandwidth to every number below. // #include @@ -142,16 +149,26 @@ static void report (const char *label, sqlite3_int64 bytes, double seconds, doub } int main (void) { + const char *dbpath = "build/benchmark.db"; + remove(dbpath); + remove("build/benchmark.db-journal"); + sqlite3 *db = NULL; - if (sqlite3_open(":memory:", &db) != SQLITE_OK) die(db, "open", NULL); + if (sqlite3_open(dbpath, &db) != SQLITE_OK) die(db, "open", NULL); if (sqlite3_vector_init(db, NULL, NULL) != SQLITE_OK) die(db, "vector_init", NULL); + // this database is a throwaway fixture, so skip the durability machinery while + // building it - it does not affect the read-only measurements below + run_sql(db, "PRAGMA journal_mode=OFF;"); + run_sql(db, "PRAGMA synchronous=OFF;"); + sqlite3_stmt *stmt = NULL; sqlite3_prepare_v2(db, "SELECT vector_backend();", -1, &stmt, NULL); sqlite3_step(stmt); printf("sqlite-vector benchmark - backend %s, SQLite %s\n", sqlite3_column_text(stmt, 0), sqlite3_libversion()); sqlite3_finalize(stmt); printf("%d vectors, dimension %d, %s distance, k=%d, %d queries, best of run\n", NVECS, DIM, DISTANCE, K, NQUERIES); + printf("file-backed database at %s, SQLite page cache left at its default\n", dbpath); printf("data is uniform random, which is the worst case for quantization recall:\n"); printf("real embeddings have structure that the quantizers exploit\n\n"); @@ -251,5 +268,6 @@ int main (void) { exact_time * 1000.0, (double)int8_bytes / (1024.0 * 1024.0)); sqlite3_close(db); + remove(dbpath); return 0; } From 5e2fb948656675388e9a3ffe305e5ca122a197e2 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Tue, 25 Aug 2026 10:00:20 +0200 Subject: [PATCH 3/3] docs: make a wrong benchmark row hard to produce Checking whether a contributor would get the numbers the table needs turned up three ways they would not. The README put "Override anything" with a worked example three lines above "it prints the two rows ready to paste". Someone runs it with NVECS=100000, pastes two rows, and the table now has entries that look comparable and are not. The benchmark now compares its parameters against the ones the table is built on and prints an explanation instead of rows when they differ. The overrides are still documented, further down, with what they are for. The backend was typed by hand as part of the HARDWARE label, so a row could claim AVX512 on a build that fell back to SSE2 - which is exactly the failure this project shipped for years. The binary now appends the backend it actually selected, and HARDWARE takes the CPU name only. HARDWARE defaulted to a placeholder that was itself paste-able. It now defaults to empty, and an unset label is reported at startup rather than after the measurements have run, so the mistake costs seconds instead of a quarter of an hour. Every figure in the section also now comes from a single run, so the hardware table and the mode table cannot drift apart. Co-Authored-By: Claude Opus 5 --- Makefile | 4 ++-- README.md | 43 ++++++++++++++++++++++++--------------- test/benchmark.c | 53 +++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 75 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index b9a4b4f..6738d67 100644 --- a/Makefile +++ b/Makefile @@ -188,14 +188,14 @@ unittest-simd: $(BUILD_DIR)/backend $(BUILD_DIR)/test_vector_simd # make benchmark k=20 over 1M vectors of dim 768 # make benchmark NVECS=100000 DIM=384 K=10 smaller, for a quick look # make benchmark DISTANCE=l2 a different metric -# make benchmark HARDWARE="M5 Pro - NEON" label the row it prints for the README +# make benchmark HARDWARE="Apple M5 Pro" name the machine; the backend is appended NVECS ?= 1000000 DIM ?= 768 K ?= 20 NQUERIES ?= 20 DISTANCE ?= cosine # names the row this run contributes to the README hardware table -HARDWARE ?= - backend +HARDWARE ?= BENCH_OBJ = $(patsubst %.c, $(BUILD_DIR)/bm-%.o, $(notdir $(SRC_FILES))) $(BUILD_DIR)/bm-sqlite3.o diff --git a/README.md b/README.md index 18f1370..6b6d6c6 100644 --- a/README.md +++ b/README.md @@ -147,27 +147,38 @@ SELECT e.id, v.distance FROM images AS e ## Benchmark -One command, so results from different machines are comparable: +To add your machine to the table, one command — pass the CPU name and nothing else: ```bash -make benchmark HARDWARE="Apple M5 Pro - NEON" +make benchmark HARDWARE="Apple M5 Pro" ``` It builds `test/benchmark.c` at `-O3` with the same per-translation-unit SIMD flags the -shipped extension uses, then runs **k=20 over 1,000,000 vectors of dimension 768** with -cosine distance, 20 queries, reporting the best. It prints the two rows below ready to -paste. Override anything: +shipped extension uses, runs **k=20 over 1,000,000 vectors of dimension 768** with cosine +distance and 20 queries reporting the best, and prints two rows ready to paste, unedited, +into the table below. + +Two things it does so a pasted row cannot be wrong. The backend is **appended by the +binary** from what the build actually selected, not typed by hand, so a row cannot claim +`AVX512` on a build that fell back to `SSE2`. And a run whose parameters differ from the +ones the table is built on **prints an explanation instead of rows**, because a row +measured on a different workload would sit in that table looking comparable without being +comparable. + +That second guard exists because the parameters *are* adjustable, just not for this table: ```bash make benchmark NVECS=100000 DIM=384 K=10 DISTANCE=l2 ``` +That run prints the mode table for whatever you asked for, and no paste-ready rows. + Common to every row: the database is **a file, never `:memory:`** — an in-memory database puts the whole index in the process no matter how it is configured, which makes any memory figure meaningless. Vectors are uniform random with a fixed seed, so two machines measure the same data. The `INT8` index is **740 MB** on disk against 2930 MB of raw `FLOAT32`. Recall is the overlap with the exact `FLOAT32` scan, the baseline everything is compared -against, 100% by definition; on the reference machine that scan takes **486 ms/query**, +against, 100% by definition; on the reference machine that scan takes **484 ms/query**, because it reads 3 GB per query. The two rows per machine are the two ways the same index gets deployed. *Preloaded* holds @@ -181,10 +192,10 @@ peak the extension and SQLite had allocated during the scan. | Hardware | Vectors | Index | Max memory | ms/query | Mvec/s | Recall@20 | | --- | ---: | --- | ---: | ---: | ---: | ---: | | Apple M5 Pro - NEON | 1,000,000 | `INT8` preloaded | 740 MB | 37.6 | 26.6 | 99.5% | -| Apple M5 Pro - NEON | 1,000,000 | `INT8` streamed | 30 MB | 115.5 | 8.7 | 99.5% | +| Apple M5 Pro - NEON | 1,000,000 | `INT8` streamed | 30 MB | 114.4 | 8.7 | 99.5% | -*Results from other CPUs welcome — run the command above and open a PR adding your two -rows.* +*Results from other CPUs welcome — run the command above and open a PR adding the two rows +it prints.* The trade is **25x less memory for 3x the latency**, with recall untouched: both rows read the same index, only how much of it is resident differs. @@ -208,12 +219,12 @@ machine, same data, all preloaded: | Mode | Index | ms/query | vs exact | Recall@20 | | --- | ---: | ---: | ---: | ---: | -| `FLOAT32` exact | 2930 MB | 486.5 | 1.0x | 100.0% | -| `UINT8` | 740 MB | 37.5 | 13.0x | 33.8% | -| `INT8` | 740 MB | 37.6 | 13.0x | 99.5% | -| `1BIT` | 99 MB | 2.7 | 183x | 10.0% | -| `TURBO2` | 195 MB | 47.8 | 10.2x | 45.2% | -| `TURBO4` | 378 MB | 150.9 | 3.2x | 81.8% | +| `FLOAT32` exact | 2930 MB | 484.4 | 1.0x | 100.0% | +| `UINT8` | 740 MB | 37.3 | 13.0x | 33.8% | +| `INT8` | 740 MB | 37.6 | 12.9x | 99.5% | +| `1BIT` | 99 MB | 2.5 | 195x | 10.0% | +| `TURBO2` | 195 MB | 48.0 | 10.1x | 45.2% | +| `TURBO4` | 378 MB | 151.5 | 3.2x | 81.8% | **The data is uniform random**, the worst case for every quantizer: real embeddings have structure quantization exploits, so recall on your own vectors will be higher, often much @@ -227,7 +238,7 @@ angle, which that shift destroys. `UINT8` is right for L2, where a common transl cancels. If you omit `qtype` the extension picks `UINT8` for non-negative data — correct for L2, wrong for cosine — so set it explicitly when you use cosine. -**`1BIT` is a filter, not an answer.** 183x faster than exact and 30x smaller, at 10% +**`1BIT` is a filter, not an answer.** 195x faster than exact and 30x smaller, at 10% recall here. It earns its place as a first pass whose survivors you re-rank at full precision. diff --git a/test/benchmark.c b/test/benchmark.c index 4f7616c..9126c96 100644 --- a/test/benchmark.c +++ b/test/benchmark.c @@ -47,9 +47,18 @@ extern int sqlite3_vector_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_ #define DISTANCE "cosine" #endif #ifndef HARDWARE -#define HARDWARE " — backend" +#define HARDWARE "" #endif +// The hardware table only means anything if every row measured the same thing, and the +// overrides above make it easy to paste a row from a different workload. These are the +// values the table is built on; a run that differs prints an explanation instead of rows. +#define TABLE_NVECS 1000000 +#define TABLE_DIM 768 +#define TABLE_K 20 +#define TABLE_NQUERIES 20 +#define TABLE_DISTANCE "cosine" + // xorshift64*: the data must be identical from run to run and from machine to machine, // and rand() is neither fast enough nor portable enough for that static uint64_t rng_state = 0x853c49e6748fea9bULL; @@ -165,10 +174,17 @@ int main (void) { sqlite3_stmt *stmt = NULL; sqlite3_prepare_v2(db, "SELECT vector_backend();", -1, &stmt, NULL); sqlite3_step(stmt); - printf("sqlite-vector benchmark - backend %s, SQLite %s\n", sqlite3_column_text(stmt, 0), sqlite3_libversion()); + char backend[32]; + snprintf(backend, sizeof(backend), "%s", (const char *)sqlite3_column_text(stmt, 0)); + printf("sqlite-vector benchmark - backend %s, SQLite %s\n", backend, sqlite3_libversion()); sqlite3_finalize(stmt); printf("%d vectors, dimension %d, %s distance, k=%d, %d queries, best of run\n", NVECS, DIM, DISTANCE, K, NQUERIES); printf("file-backed database at %s, SQLite page cache left at its default\n", dbpath); + if (HARDWARE[0] == 0) { + printf("\nNOTE: HARDWARE is not set, so this run will not print rows for the README\n"); + printf(" table. Stop now and re-run as: make benchmark HARDWARE=\"Apple M5 Pro\"\n"); + printf(" Give the CPU only - the backend (%s here) is appended automatically.\n", backend); + } printf("data is uniform random, which is the worst case for quantization recall:\n"); printf("real embeddings have structure that the quantizers exploit\n\n"); @@ -255,14 +271,37 @@ int main (void) { // The README table compares hardware, so it carries the two configurations that // differ by deployment rather than by accuracy: the whole index in RAM, and the same // index streamed in 30 MB. Everything else about INT8 is a property of the data. + printf("\n"); + int canonical = (NVECS == TABLE_NVECS) && (DIM == TABLE_DIM) && (K == TABLE_K) && + (NQUERIES == TABLE_NQUERIES) && (strcmp(DISTANCE, TABLE_DISTANCE) == 0); + if (!canonical) { + printf("These parameters are not the ones the README hardware table is built on, so\n"); + printf("no rows are printed - a row measured on a different workload would sit in that\n"); + printf("table looking comparable without being comparable.\n\n"); + printf(" this run NVECS=%d DIM=%d K=%d NQUERIES=%d DISTANCE=%s\n", NVECS, DIM, K, NQUERIES, DISTANCE); + printf(" the table NVECS=%d DIM=%d K=%d NQUERIES=%d DISTANCE=%s\n\n", + TABLE_NVECS, TABLE_DIM, TABLE_K, TABLE_NQUERIES, TABLE_DISTANCE); + printf("For the table, run: make benchmark HARDWARE=\"\"\n"); + sqlite3_close(db); + remove(dbpath); + return 0; + } + if (HARDWARE[0] == 0) { + printf("No rows printed: HARDWARE was not set. Re-run as\n\n"); + printf(" make benchmark HARDWARE=\"Apple M5 Pro\"\n"); + sqlite3_close(db); + remove(dbpath); + return 0; + } + char nbuf[32]; with_separators(NVECS, nbuf, sizeof(nbuf)); - printf("\n\nPaste these two rows into the hardware table in README.md:\n\n"); - printf("| %s | %s | `INT8` preloaded | %.0f MB | %.1f | %.1f | %.1f%% |\n", - HARDWARE, nbuf, (double)int8_pre_mem / (1024.0 * 1024.0), + printf("Paste these two rows into the hardware table in README.md:\n\n"); + printf("| %s - %s | %s | `INT8` preloaded | %.0f MB | %.1f | %.1f | %.1f%% |\n", + HARDWARE, backend, nbuf, (double)int8_pre_mem / (1024.0 * 1024.0), int8_pre_ms * 1000.0, NVECS / int8_pre_ms / 1e6, int8_recall); - printf("| %s | %s | `INT8` streamed | %.0f MB | %.1f | %.1f | %.1f%% |\n", - HARDWARE, nbuf, (double)int8_stream_mem / (1024.0 * 1024.0), + printf("| %s - %s | %s | `INT8` streamed | %.0f MB | %.1f | %.1f | %.1f%% |\n", + HARDWARE, backend, nbuf, (double)int8_stream_mem / (1024.0 * 1024.0), int8_stream_ms * 1000.0, NVECS / int8_stream_ms / 1e6, int8_recall); printf("\nreference for this machine: FLOAT32 exact %.1f ms/query, index on disk %.0f MB\n", exact_time * 1000.0, (double)int8_bytes / (1024.0 * 1024.0));