diff --git a/.github/workflows/turbovec9-lab.yml b/.github/workflows/turbovec9-lab.yml index 6a54acb3..26e86db5 100644 --- a/.github/workflows/turbovec9-lab.yml +++ b/.github/workflows/turbovec9-lab.yml @@ -7,6 +7,7 @@ on: - "turbovec/Cargo.toml" - "turbovec/src/codebook.rs" - "docs/turbovec9-experiment.md" + - "docs/turbovec9-learning-neighbor-probe.md" - ".github/workflows/turbovec9-lab.yml" permissions: @@ -54,6 +55,21 @@ jobs: cargo run --release -p turbovec --features turbovec9-lab --example turbovec9_probe | tee turbovec9-probe-results-large.txt cat turbovec9-probe-results-large.txt >> "$GITHUB_STEP_SUMMARY" + - name: Graph-neighbor retention (independent edge labels) + working-directory: turbovec + env: + TV9_WORKLOAD: "graph" + TV9_N: "2048" + TV9_NQ: "64" + TV9_DIM: "128" + TV9_K: "10" + run: | + : > turbovec9-graph-results.txt + for noise in 0.8 1.7 2.5; do + TV9_NOISE="$noise" cargo run --release -p turbovec --features turbovec9-lab --example turbovec9_probe | tee -a turbovec9-graph-results.txt + done + cat turbovec9-graph-results.txt >> "$GITHUB_STEP_SUMMARY" + - name: Save raw probe output uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -61,3 +77,4 @@ jobs: path: | turbovec/turbovec9-probe-results.txt turbovec/turbovec9-probe-results-large.txt + turbovec/turbovec9-graph-results.txt diff --git a/docs/turbovec9-learning-neighbor-probe.md b/docs/turbovec9-learning-neighbor-probe.md new file mode 100644 index 00000000..95d53d00 --- /dev/null +++ b/docs/turbovec9-learning-neighbor-probe.md @@ -0,0 +1,120 @@ +# TurboVec9 × structured learning: graph-neighbor selection probe + +**2026-10-29 · lab only · no production codec or index changes** + +Why: NSL graph regularization is only as useful as the neighbor features/edges +it receives. TurboVec9's first scalar experiment showed high Recall@10 on +random unit vectors, but **did not measure labeled neighbor retention**. +This experiment tests that missing, narrower premise, not training quality. + +## Oracle and graph + +- **Truth graph is independent of quantization:** a seeded center is mixed + into four database observations (same group label). Each query is a separate + noisy observation of one center. The four group members are its *explicit + graph edges*, even when exact cosine retrieval would choose another point. +- **f32 search oracle:** deterministic exact f64 dot-product ranking on the + original f32 vectors. This is a retrieval reference, **not a graph-label + oracle**, which is why both metrics are reported separately. +- **Controls:** scalar i4 and scalar i9 both use the **same upstream Lloyd-Max + quantizer and deterministic two-round block-Hadamard rotation**. The shipped + TurboQuantIndex i4 is listed separately because it has TQ+ calibration and + SIMD/search policies that do not match the scalar controls. +- **Outputs:** (1) recall@k vs f32 top-k; (2) proportion of queries with + at least one genuine group neighbor in top-k; (3) fraction of the four + explicit edges found per query. These are *graph-construction/neighbor + sampling proxies*; no gradients or neural training are executed. +- **Crucial limit:** improved top-k ranking is not guaranteed to improve + NSL classification, graph loss, or robustness. Graph edges need an + independently justified semantic source. + +## Run + +```sh +TV9_WORKLOAD=graph cargo run --release -p turbovec --features turbovec9-lab --example turbovec9_probe + +TV9_WORKLOAD=graph TV9_N=2048 TV9_NQ=64 TV9_DIM=128 \ +TV9_K=10 TV9_NOISE=1.7 cargo run --release -p turbovec \ + --features turbovec9-lab --example turbovec9_probe +``` + +Change `TV9_NOISE` to stress neighborhood separation; use identical seeds +and dimensions across controls. Payload counts only bit-packed vector codes, +not centroid tables, scale metadata or search caches. + +## Cross-repo reuse (no copying) + +1. **Numeric 9-bit**: `ndarray::hpc::codec9` owns DR512, + `P=0,M=0` = never touched and touched zero is `+0`. + It is **not** a 512-centroid vector quantizer. Neither codeword maps nor + NaN policy may be conflated with TurboVec9. +2. **Neighbor indexing**: `turbovec` owns trained-data-oblivious + Lloyd-Max centroid selection, packed code layout, search and ID/filter. + It does *not* store NSL edge truth. +3. **Graph/loss oracle**: `lance-graph-nsl` owns the frozen Google + graph-loss fixtures. It should consume selected neighbor IDs or an + explicitly described feature view; it should not reimplement Lloyd-Max. +4. **Other existing substrates**: `ndarray::hpc::edge_codec` owns + CoarseResidue/PQ32x4, `ndarray::hpc::cam_pq` owns PQ-ADC cascade. + `bgz-tensor::turboquant_kv` owns the independent gain-shape i4 + cache probe. They are related *alternatives*, not byte-compatible codecs. + +## Falsifiers and next admission + +- If i9 graph-edge retention is not better at a useful cost, no promotion. +- Compare **fixed-budget** recall, bytes/query, and runtime on real embeddings + before making a recommendation; do not infer relative performance from + scalar i9 vs SIMD i4. +- Next: NSL loss/gradient/epoch parity on *fixed* graph labels, then *separate* + learned graph construction using i4 vs i9 retrieved neighbor lists; report + graph-edge precision and class accuracy under held-out labels. +- QJL residual debiasing is separate from the i9 test. The Google TurboQuant + paper's two-stage method does not become proven merely by 9-bit Lloyd-Max. +- Do not silently coerce DR512 never-touched lanes to valid zeros in a + learning input pipeline. + +References: +- https://arxiv.org/abs/2504.19874 +- https://research.google/blog/turboquant-redefining-ai-efficiency-with-extreme-compression/ +- https://github.com/tensorflow/neural-structured-learning +- https://github.com/AdaWorldAPI/ndarray/blob/master/src/hpc/codec9.rs +- https://github.com/AdaWorldAPI/ndarray/blob/master/src/hpc/edge_codec.rs +- https://github.com/AdaWorldAPI/lance-graph/tree/main/crates/lance-graph-nsl + +## First measured graph-selection results (GitHub Actions, 2026-10-29) + +CI run: https://github.com/AdaWorldAPI/turbovec/actions/runs/40880844314 +(`cargo test`: **5 passed**, all three graph noise cases executed). +Dataset: 2048 vectors, 128-D, 64 queries, top-10, 4 explicit neighbors per +query. Fixed seed. **None of these metrics are a training result.** + +| noise | path | f32-top10 recall | any graph-neighbor@10 | explicit graph-edge coverage@10 | +|---:|---|---:|---:|---:| +| 0.8 | f32 | 1.00000 | 1.00000 | 1.00000 | +| | scalar i4 | 0.77812 | 1.00000 | 1.00000 | +| | scalar i9 | 0.98125 | 1.00000 | 1.00000 | +| | shipped i4 | 0.78437 | 1.00000 | 1.00000 | +| 1.7 | f32 | 1.00000 | 0.89062 | 0.48438 | +| | scalar i4 | 0.65469 | 0.87500 | 0.42578 | +| | scalar i9 | 0.97187 | 0.89062 | 0.47656 | +| | shipped i4 | 0.66875 | 0.87500 | 0.42969 | +| 2.5 | f32 | 1.00000 | 0.18750 | 0.06250 | +| | scalar i4 | 0.67969 | 0.23438 | 0.06641 | +| | scalar i9 | 0.97031 | 0.20312 | 0.07031 | +| | shipped i4 | 0.67813 | 0.23438 | 0.06641 | + +**What this actually falsifies:** high f32-neighbor recall is *not* +equivalent to high semantic graph-edge recall. At noise=0.8 all paths +retrieve all four ground-truth neighbors despite large i4/f32 top-k +differences. At noise=2.5, i4 sometimes hits more explicit graph labels +than f32 by approximation noise; `f32` is a vector-similarity oracle +and **not a semantic upper bound**. At noise=1.7, i9 improves explicit +edge coverage by 0.05078 absolute over scalar i4, but the any-positive +rate improves by only 0.01562. + +**Cost qualification:** the 9-bit scalar scan is about 2x the +4-bit scalar scan, with 2.25x the vector payload; shipped i4 is much +faster due to a different SIMD/calibration implementation. Absolute +times are GitHub-runner-dependent. Do not promote U9 for NSL on this +sample; next evaluate task labels, graph regularization, gradient +fidelity, and actual training accuracy on realistic graph data. diff --git a/turbovec/examples/turbovec9_probe.rs b/turbovec/examples/turbovec9_probe.rs index 15f420de..484c2808 100644 --- a/turbovec/examples/turbovec9_probe.rs +++ b/turbovec/examples/turbovec9_probe.rs @@ -5,10 +5,12 @@ //! The 9-bit code is a 0..511 Lloyd-Max centroid ID, NOT ndarray Codec9's //! DR512 presence/polarity encoding. //! -//! From the AdaWorldAPI/turbovec checkout with sibling ../ndarray: -//! cargo test -p turbovec --release --example turbovec9_probe -//! cargo run -p turbovec --release --example turbovec9_probe -//! Environment overrides: TV9_N, TV9_NQ, TV9_DIM, TV9_K, TV9_REPEATS. +//! From the standalone AdaWorldAPI/turbovec checkout: +//! cargo test --release -p turbovec --features turbovec9-lab --example turbovec9_probe +//! cargo run --release -p turbovec --features turbovec9-lab --example turbovec9_probe +//! TV9_WORKLOAD=graph cargo run --release -p turbovec --features turbovec9-lab --example turbovec9_probe +//! Environment overrides: TV9_N, TV9_NQ, TV9_DIM, TV9_K, TV9_REPEATS, +//! TV9_WORKLOAD, TV9_NOISE (only for the graph workload). use std::hint::black_box; use std::time::Instant; @@ -171,11 +173,125 @@ fn recall(found: &[usize], truth: &[usize]) -> f64 { hits as f64 / truth.len().max(1) as f64 } +/// Graph-regularization **input** probe: each group of four database +/// vectors is an explicit ground-truth neighborhood, independent of the +/// retrieval encoder. A query is a separate noisy observation of one group. +/// This measures neighbor selection only, NOT backprop/training accuracy. +fn graph_vectors(n: usize, nq: usize, dim: usize, noise: f32) -> (Vec, Vec) { + assert!(n >= 4 && n % 4 == 0 && nq > 0 && dim % 8 == 0); + assert!(noise >= 0.0 && noise.is_finite()); + let groups = n / 4; + let centers = unit_vectors(groups, dim, 0xD0C); + let perturb_db = unit_vectors(n, dim, 0xBAD); + let perturb_q = unit_vectors(nq, dim, 0xF00); + let mut db = vec![0.0; n * dim]; + let mut queries = vec![0.0; nq * dim]; + fn mix(dst: &mut [f32], center: &[f32], jitter: &[f32], noise: f32) { + let mut norm = 0f64; + for d in 0..dst.len() { + dst[d] = center[d] + noise * jitter[d]; + norm += (dst[d] as f64) * (dst[d] as f64); + } + let inv = norm.sqrt().recip() as f32; + for x in dst { *x *= inv; } + } + for row in 0..n { + let group = row / 4; + mix( + &mut db[row * dim..(row + 1) * dim], + ¢ers[group * dim..(group + 1) * dim], + &perturb_db[row * dim..(row + 1) * dim], + noise, + ); + } + for q in 0..nq { + let group = q % groups; + mix( + &mut queries[q * dim..(q + 1) * dim], + ¢ers[group * dim..(group + 1) * dim], + &perturb_q[q * dim..(q + 1) * dim], + noise, + ); + } + (db, queries) +} + +/// Independent explicit graph: group q%groups has exactly four edges. +/// "hit" means any graph edge appears in top-k, "coverage" counts all four. +fn graph_edges_at_k(found: &[usize], query: usize, groups: usize) -> (bool, f64) { + let group = query % groups; + let relevant = (group * 4)..(group * 4 + 4); + let hits = found.iter().filter(|&&i| relevant.contains(&i)).count(); + (hits > 0, hits as f64 / 4.0) +} + +fn run_graph_probe() { + let n = env_usize("TV9_N", 512); + let nq = env_usize("TV9_NQ", 64); + let dim = env_usize("TV9_DIM", 64); + let k = env_usize("TV9_K", 10).min(n); + let noise: f32 = std::env::var("TV9_NOISE") + .ok().and_then(|s| s.parse().ok()).unwrap_or(1.7); + assert!(n >= 4 && n % 4 == 0 && dim > 0 && dim % 8 == 0 && nq > 0 && k > 0); + let (db, queries) = graph_vectors(n, nq, dim, noise); + let groups = n / 4; + let rotation = Rotation::new(dim); + let i4 = ScalarTurbo::encode(&db, dim, 4, &rotation).expect("4-bit"); + let i9 = ScalarTurbo::encode(&db, dim, 9, &rotation).expect("9-bit"); + let mut prod4 = TurboQuantIndex::new(dim, 4).expect("shipped 4-bit"); + prod4.add(&db); + prod4.prepare(); + let truth: Vec> = queries.chunks_exact(dim) + .map(|q| exact_topk(&db, q, dim, k)).collect(); + + println!("TurboVec9 GRAPH-LAB | n={n} dim={dim} queries={nq} k={k} noise={noise} | 4 labeled positives/group"); + println!("METRICS: topk_recall_f32 / graph_any_positive@k / graph_edge_coverage@k"); + for (name, bits, search) in [ + ("f32-exact", 32usize, 0usize), + ("scalar-i4", 4, 1), + ("scalar-i9", 9, 2), + ("shipped-i4", 4, 3), + ] { + let mut recall_sum = 0.0f64; + let mut hit_sum = 0usize; + let mut edge_sum = 0.0f64; + let t = Instant::now(); + // This loop is the same for all encoders; production search + // includes its own per-query dispatch and index lookup. + for (qi, query) in queries.chunks_exact(dim).enumerate() { + let top = match search { + 0 => exact_topk(&db, query, dim, k), + 1 => i4.topk(query, &rotation, k), + 2 => i9.topk(query, &rotation, k), + _ => prod4.search(query, k).indices_for_query(0) + .iter().map(|&i| i as usize).collect(), + }; + recall_sum += recall(&top, &truth[qi]); + let (hit, coverage) = graph_edges_at_k(&top, qi, groups); + hit_sum += hit as usize; + edge_sum += coverage; + } + println!( + "{name:10} payload={:4} B/vector recall_f32@{k}={:.5} graph_any@{k}={:.5} graph_coverage@{k}={:.5} query={:.1} us", + if search == 0 { dim * 4 } else { row_bytes(dim, bits) }, + recall_sum / nq as f64, + hit_sum as f64 / nq as f64, + edge_sum / nq as f64, + t.elapsed().as_secs_f64() * 1e6 / nq as f64, + ); + } + println!("Interpretation: neighbor-selection proxy only; no NSL graph loss/gradient/training run."); +} + fn env_usize(name: &str, default: usize) -> usize { std::env::var(name).ok().and_then(|s| s.parse().ok()).unwrap_or(default) } fn main() { + if std::env::var("TV9_WORKLOAD").ok().as_deref() == Some("graph") { + run_graph_probe(); + return; + } let n = env_usize("TV9_N", 512); let nq = env_usize("TV9_NQ", 32); let dim = env_usize("TV9_DIM", 64); @@ -277,6 +393,32 @@ mod tests { assert!(top.iter().all(|&id| id < 24)); } + #[test] + fn four_explicit_graph_edges_and_query_groups_are_stable() { + let (db, q) = graph_vectors(24, 7, 32, 1.7); + assert_eq!(db.len(), 24 * 32); + assert_eq!(q.len(), 7 * 32); + assert_eq!(graph_edges_at_k(&[0, 3, 8], 0, 6), (true, 0.5)); + assert_eq!(graph_edges_at_k(&[0, 3, 8], 1, 6), (false, 0.0)); + assert_eq!(graph_edges_at_k(&[0, 3, 8], 6, 6), (true, 0.5)); + } + + #[test] + fn graph_probe_retrieval_has_valid_metrics_and_no_duplicate_ids() { + let (db, queries) = graph_vectors(32, 5, 32, 1.7); + let r = Rotation::new(32); + let ix = ScalarTurbo::encode(&db, 32, 9, &r).unwrap(); + for (i, query) in queries.chunks_exact(32).enumerate() { + let top = ix.topk(query, &r, 10); + assert_eq!(top.len(), 10); + let unique: std::collections::BTreeSet = top.iter().copied().collect(); + assert_eq!(unique.len(), top.len()); + let (hit, coverage) = graph_edges_at_k(&top, i, 8); + assert_eq!(hit, coverage > 0.0); + assert!((0.0..=1.0).contains(&coverage)); + } + } + #[test] fn nullable_values_must_refuse_before_encoding() { let dim = 8;