From a08e60f49bf9aa9538eeb34bd6e8088cb9808399 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 17:51:35 +0000 Subject: [PATCH 1/9] dr-isa-stream-lab: INT8 direct arm on ndarray's prepacked INT8 entries Adds int8::direct and int8::PreparedI8B. A is read in place at the u8 lanes' pitch and B is prepared once as PackedI8B bands, through ndarray::simd::int8_gemm_packed_strided (AdaWorldAPI/ndarray#418). The prepared-B reuse rule moves into shape::ReuseKey, shared by the BF16 and INT8 prepared operands so the two cannot drift. Measured on one AMX VM (median of 41 warm runs, three runs), 256 fields x 256 columns: the direct GEMM takes 45.5-48.1 us against 78.3-78.8 us for backend::gemm_i8, end to end 47.5-57.9 us against 88.9-92.2 us; at 64 fields 0.98-1.01 us against 3.4-3.6 us. No A copied, no allocation. Needs ndarray#418. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .../shape-2026-11-10-int8-direct-run1.txt | 33 ++++ .../shape-2026-11-10-int8-direct-run2.txt | 33 ++++ .../shape-2026-11-10-int8-direct-run3.txt | 33 ++++ crates/dr-isa-stream-lab/src/bin/shape.rs | 37 ++++ crates/dr-isa-stream-lab/src/int8.rs | 179 +++++++++++++++++- crates/dr-isa-stream-lab/src/shape.rs | 81 +++++--- .../tests/integration/int8.rs | 94 +++++++++ 7 files changed, 454 insertions(+), 36 deletions(-) create mode 100644 crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run1.txt create mode 100644 crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run2.txt create mode 100644 crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run3.txt diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run1.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run1.txt new file mode 100644 index 000000000..c1173d23d --- /dev/null +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run1.txt @@ -0,0 +1,33 @@ +Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K +Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). + +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 382.6 µs + +| Metric | 64×64 | 256×256 | +|---|---|---| +| Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | +| DT64 bind (cold) | 3187 ns · 2 allocs | 8623 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1256 ns · 0 allocs · 0 B | 1983 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.7 µs | 146.5 µs | +| B preparation (VNNI panels) | 6703 ns · 8192 B written · 8320 B alloc | 98.7 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4306 ns · 60.9 GMAC/s · 0 allocs · 0 B tmp | 137.3 µs · 122.2 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 3759 ns · 69.7 GMAC/s | 120.7 µs · 139.0 GMAC/s | +| Staged AMX (dense A tiles) | 4140 ns · 1 allocs · 2048 B tmp | 127.7 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 36.4 µs · 23 allocs · 54272 B tmp | 1081.9 µs · 263 allocs · 803840 B tmp | +| Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | +| CBLAS B materialized | 8192 B | 131072 B | +| FP32 accumulator (C) | 16384 B | 262144 B | +| End-to-end warm (resolve + direct, B reused) | 6528 ns | 147.9 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 23.0 µs | 303.5 µs | +| Warm repetitions (100, per run, one prepare) | 7019 ns | 154.4 µs | +| INT8 resolve (u8 lanes, one check) | 1213 ns | 1960 ns | +| INT8 stage A (region: one copy per lane) | 310 ns · 4096 B | 4700 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3312 ns | 30.8 µs | +| INT8 B for a region, in place from the table | 30 ns · 0 B written | 30 ns · 0 B written | +| INT8 materialize B (row-major i8) | 7564 ns · 4096 B | 124.1 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3552 ns · 73.8 GMAC/s · 3 allocs · 3072 B tmp | 78.3 µs · 214.3 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 2009 ns · 57% of the GEMM row | 29.4 µs · 38% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 5856 ns | 88.9 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 6637 ns · 4096 B | 105.3 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 1011 ns · 259.3 GMAC/s · 0 allocs · 0 B of A copied | 48.1 µs · 348.5 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 3013 ns | 57.9 µs | diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run2.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run2.txt new file mode 100644 index 000000000..a76b40cca --- /dev/null +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run2.txt @@ -0,0 +1,33 @@ +Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K +Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). + +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 385.0 µs + +| Metric | 64×64 | 256×256 | +|---|---|---| +| Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | +| DT64 bind (cold) | 3088 ns · 2 allocs | 8409 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1181 ns · 0 allocs · 0 B | 2488 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.3 µs | 181.1 µs | +| B preparation (VNNI panels) | 6570 ns · 8192 B written · 8320 B alloc | 110.5 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4157 ns · 63.1 GMAC/s · 0 allocs · 0 B tmp | 123.2 µs · 136.2 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 3701 ns · 70.8 GMAC/s | 119.5 µs · 140.4 GMAC/s | +| Staged AMX (dense A tiles) | 4007 ns · 1 allocs · 2048 B tmp | 113.1 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 35.7 µs · 23 allocs · 54272 B tmp | 1184.9 µs · 263 allocs · 803840 B tmp | +| Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | +| CBLAS B materialized | 8192 B | 131072 B | +| FP32 accumulator (C) | 16384 B | 262144 B | +| End-to-end warm (resolve + direct, B reused) | 6254 ns | 131.1 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 21.9 µs | 274.3 µs | +| Warm repetitions (100, per run, one prepare) | 6777 ns | 145.2 µs | +| INT8 resolve (u8 lanes, one check) | 1151 ns | 1979 ns | +| INT8 stage A (region: one copy per lane) | 302 ns · 4096 B | 4704 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3369 ns | 30.4 µs | +| INT8 B for a region, in place from the table | 29 ns · 0 B written | 30 ns · 0 B written | +| INT8 materialize B (row-major i8) | 7343 ns · 4096 B | 125.3 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3432 ns · 76.4 GMAC/s · 3 allocs · 3072 B tmp | 78.8 µs · 212.8 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 2149 ns · 63% of the GEMM row | 29.4 µs · 37% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 5745 ns | 89.0 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 6444 ns · 4096 B | 105.8 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 976 ns · 268.6 GMAC/s · 0 allocs · 0 B of A copied | 47.3 µs · 354.7 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 2924 ns | 52.7 µs | diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run3.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run3.txt new file mode 100644 index 000000000..30aa19037 --- /dev/null +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-int8-direct-run3.txt @@ -0,0 +1,33 @@ +Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K +Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). + +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 388.9 µs + +| Metric | 64×64 | 256×256 | +|---|---|---| +| Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | +| DT64 bind (cold) | 3179 ns · 2 allocs | 8882 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1210 ns · 0 allocs · 0 B | 1991 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.7 µs | 151.3 µs | +| B preparation (VNNI panels) | 6703 ns · 8192 B written · 8320 B alloc | 109.0 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4314 ns · 60.8 GMAC/s · 0 allocs · 0 B tmp | 170.5 µs · 98.4 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 3804 ns · 68.9 GMAC/s | 152.2 µs · 110.2 GMAC/s | +| Staged AMX (dense A tiles) | 4232 ns · 1 allocs · 2048 B tmp | 146.4 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 36.8 µs · 23 allocs · 54272 B tmp | 1527.3 µs · 263 allocs · 803840 B tmp | +| Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | +| CBLAS B materialized | 8192 B | 131072 B | +| FP32 accumulator (C) | 16384 B | 262144 B | +| End-to-end warm (resolve + direct, B reused) | 6474 ns | 145.1 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 18.3 µs | 295.9 µs | +| Warm repetitions (100, per run, one prepare) | 6384 ns | 169.6 µs | +| INT8 resolve (u8 lanes, one check) | 1188 ns | 1941 ns | +| INT8 stage A (region: one copy per lane) | 309 ns · 4096 B | 4742 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3424 ns | 29.3 µs | +| INT8 B for a region, in place from the table | 30 ns · 0 B written | 30 ns · 0 B written | +| INT8 materialize B (row-major i8) | 7569 ns · 4096 B | 125.4 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3553 ns · 73.8 GMAC/s · 3 allocs · 3072 B tmp | 78.5 µs · 213.8 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 1909 ns · 54% of the GEMM row | 28.6 µs · 37% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 5878 ns | 92.2 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 6659 ns · 4096 B | 105.4 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 1006 ns · 260.6 GMAC/s · 0 allocs · 0 B of A copied | 45.5 µs · 368.3 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 3035 ns | 47.5 µs | diff --git a/crates/dr-isa-stream-lab/src/bin/shape.rs b/crates/dr-isa-stream-lab/src/bin/shape.rs index 826cee7c2..96feaf578 100644 --- a/crates/dr-isa-stream-lab/src/bin/shape.rs +++ b/crates/dr-isa-stream-lab/src/bin/shape.rs @@ -397,6 +397,24 @@ fn main() { }); assert_eq!(c8, want8); let (a8_allocs, m8_gemm, _) = mem(|| int8::gemm(&o8, &a8, &b8, &mut c8)); + // The direct arm: A read in place at the lane pitch, B prepared once + // into PackedI8B bands (ndarray's INT8 twins of #414). + let prep8 = int8::PreparedI8B::prepare(&o8, &bt).unwrap(); + let mut d8 = vec![0i32; o8.c_len()]; + int8::direct(&o8, &l8, &prep8, &bt, &mut d8).unwrap(); + assert_eq!(d8, want8, "int8 direct {dim}"); + let t8_prep = median_ns(|| { + black_box(int8::PreparedI8B::prepare(&o8, &bt).unwrap()); + }); + let t8_direct = median_ns(|| { + int8::direct(&o8, &l8, &prep8, &bt, &mut d8).unwrap(); + }); + let t8_direct_all = median_ns(|| { + let l = int8::resolve(&o8, &world, &u8s).unwrap(); + int8::direct(&o8, &l, &prep8, &bt, &mut d8).unwrap(); + }); + assert_eq!(d8, want8); + let (d8_allocs, _, _) = mem(|| int8::direct(&o8, &l8, &prep8, &bt, &mut d8).unwrap()); push("INT8 resolve (u8 lanes, one check)", col, fmt(t8_resolve)); push( "INT8 stage A (region: one copy per lane)", @@ -441,6 +459,25 @@ fn main() { col, fmt(t8_all), ); + push( + "INT8 prepare B once (PackedI8B bands, `ndarray::simd`)", + col, + format!("{} · {} B", fmt(t8_prep), prep8.bytes()), + ); + push( + "INT8 direct GEMM (A in place, prepared B)", + col, + format!( + "{} · {:.1} GMAC/s · {d8_allocs} allocs · 0 B of A copied", + fmt(t8_direct), + macs / t8_direct + ), + ); + push( + "INT8 direct end-to-end warm (resolve + GEMM)", + col, + fmt(t8_direct_all), + ); } // Sparse demand: 40 scattered identities, 256 fields x 256 columns. diff --git a/crates/dr-isa-stream-lab/src/int8.rs b/crates/dr-isa-stream-lab/src/int8.rs index fd0664854..39c0e8444 100644 --- a/crates/dr-isa-stream-lab/src/int8.rs +++ b/crates/dr-isa-stream-lab/src/int8.rs @@ -8,13 +8,16 @@ //! The binding is the same [`VirtualObject`]: DT64 does not care what a lane //! holds. Only the borrow and the arithmetic change. //! -//! What ndarray's facade offers for INT8 today (audited 2026-11-10): -//! `backend::gemm_i8` -> `simd_int_ops::gemm_u8_i8` -> `int8_gemm_amx_tiled` -//! on 16/16/64-aligned shapes. A must be dense with row stride K, and B is -//! VNNI-packed inside every call. There is no strided-A entry and no prepacked -//! B for INT8 (the BF16 equivalents are `bf16_tile_gemm_16x16_packed_strided` -//! and `PackedBf16B`). So the only INT8 arm here is [`staged`]: a direct -//! (in-place) arm and prepared-B reuse are refused, not emulated. +//! Two INT8 arms: +//! - [`staged`]: `backend::gemm_i8` (-> `int8_gemm_amx_tiled`). A is copied +//! dense with row stride K, and B is gathered and VNNI-packed inside every +//! call. +//! - [`direct`]: A read in place at the lanes' pitch through +//! `ndarray::simd::int8_gemm_packed_strided`, and B prepared +//! once ([`PreparedI8B`], one `PackedI8B` band per 16 output columns) and +//! reused under the same identity rule as the BF16 [`crate::shape::PreparedB`] +//! ([`ReuseKey`]). These are the INT8 twins of the BF16 entries ndarray +//! added in #414. use std::cell::Cell; @@ -22,9 +25,15 @@ use lance_graph_contract::ontology::FieldRef; use lance_graph_dt64::dr::{Frame, FrameRevisions, FrameStore, Scope}; use lance_graph_dt64::{CoordSpace, LaneAddr, LaneRefusal, LaneSource, SchemaRev}; +use ndarray::simd::{int8_gemm_packed_strided, PackedI8B}; + use crate::bind::{World, CLASS, READER, SPACE_A}; use crate::fixture::Fixture; -use crate::shape::{BOperand, BValues, Selection, ShapeRefusal, VirtualObject, Where, MAX_ROWS}; +use crate::gemm::DirectRefusal; +use crate::shape::{ + admit_operand, BOperand, BValues, ReuseKey, Selection, ShapeRefusal, VirtualObject, Where, + MAX_ROWS, TILE, +}; /// INT8 tiles consume K in steps of 64. pub const K_STEP: usize = 64; @@ -365,3 +374,157 @@ pub fn oracle(fx: &Fixture, sel: &Selection, fields: &[u8], cols: usize, b: &BOp }); c } + +/// B as `cols / 16` VNNI-packed `K x 16` i8 bands, prepared once for one +/// object and one operand identity, reused by every output tile and every +/// repeated run. Rows are the selected identities in order; rows past the +/// selection are zero. +pub struct PreparedI8B { + key: ReuseKey, + bands: Vec, +} + +impl PreparedI8B { + /// Pack B for `obj`: `band t row k = B[i_k][16t..16t+16]` in VNNI order. + /// Refuses an operand smaller than the object ([`admit_operand`]). + pub fn prepare(obj: &VirtualObject, b: &impl BValues) -> Result { + admit_operand(obj, b)?; + let k_pad = obj.selection().len().div_ceil(K_STEP) * K_STEP; + let mut bands: Vec = (0..obj.cols() / TILE) + .map(|_| PackedI8B::zeroed(k_pad)) + .collect(); + obj.selection().for_each(|row, i| { + for (t, band) in bands.iter_mut().enumerate() { + let data = band.data_mut(); + for c in 0..TILE { + data[PackedI8B::vnni_index(row, c)] = b.i8_at(i, t * TILE + c); + } + } + }); + Ok(PreparedI8B { + key: ReuseKey::of(obj, b), + bands, + }) + } + + /// Bytes the bands occupy, K padding included. + pub fn bytes(&self) -> usize { + self.bands.iter().map(|b| b.data().len()).sum() + } +} + +/// Where 16 lanes sit when they can be read as one strided A operand. +struct Geometry<'s> { + slab: &'s [u8], + pitch: usize, + first_row: usize, +} + +/// The 16 lanes of a row tile as one strided operand: one allocation, one +/// pitch, consecutive rows. Anything else is refused, never copied. +fn geometry<'s>(fields: &[u8], lanes: &[U8Lane<'s>]) -> Result, DirectRefusal> { + if lanes.len() < TILE { + return Err(DirectRefusal::PartialRowTile { lanes: lanes.len() }); + } + let first = lanes[0]; + for (k, (lane, &field)) in lanes.iter().zip(fields).enumerate().skip(1) { + if !std::ptr::eq(lane.slab, first.slab) { + return Err(DirectRefusal::NotOneSlab { field }); + } + if lane.pitch != first.pitch { + return Err(DirectRefusal::PitchMismatch { field }); + } + if lane.row != first.row + k { + return Err(DirectRefusal::RowsNotConsecutive { field }); + } + } + Ok(Geometry { + slab: first.slab, + pitch: first.pitch, + first_row: first.row, + }) +} + +/// The INT8 direct arm: A read in place at the lanes' pitch, B from the +/// prepared bands. Needs a region selection; `c` is `rows x cols`, row-major, +/// overwritten. Rows of B past the selection are zero, so the read up to the +/// 64-step padding adds nothing; the read must still stay inside the lane's +/// own row of the slab. +pub fn direct( + obj: &VirtualObject, + lanes: &U8Lanes64<'_>, + b: &PreparedI8B, + operand: &impl BValues, + c: &mut [i32], +) -> Result { + b.key.admit(obj, operand).map_err(ShapeRefusal::Reuse)?; + admit(obj, lanes)?; + let Where::Region { start, len } = obj.selection().kind else { + return Err(ShapeRefusal::NotARegion); + }; + let (rows, cols) = (obj.fields().len(), obj.cols()); + assert_eq!(c.len(), rows * cols); + let k_pad = len.div_ceil(K_STEP) * K_STEP; + let mut report = Int8Report { + k_pad, + ..Int8Report::default() + }; + // One call when every lane is one strided operand (the source's own + // order); otherwise one call per 16-field group, each writing its rows. + if geometry(obj.fields(), lanes.lanes()).is_ok() { + run_rows( + start, + len, + k_pad, + 0, + obj.fields(), + lanes.lanes(), + b, + c, + cols, + &mut report, + )?; + } else { + for (g, (fs, ls)) in obj + .fields() + .chunks(TILE) + .zip(lanes.lanes().chunks(TILE)) + .enumerate() + { + run_rows(start, len, k_pad, g * TILE, fs, ls, b, c, cols, &mut report)?; + } + } + Ok(report) +} + +/// One strided GEMM over the lanes `ls` (fields `fs`), writing C's rows from +/// `row0`. +#[allow(clippy::too_many_arguments)] +fn run_rows( + start: usize, + len: usize, + k_pad: usize, + row0: usize, + fs: &[u8], + ls: &[U8Lane<'_>], + b: &PreparedI8B, + c: &mut [i32], + cols: usize, + report: &mut Int8Report, +) -> Result<(), ShapeRefusal> { + let geo = geometry(fs, ls).map_err(ShapeRefusal::Direct)?; + if start + k_pad > geo.pitch { + return Err(ShapeRefusal::OverRead { + start, + len, + k_pad, + n: ls[0].n, + pitch: geo.pitch, + }); + } + let a = &geo.slab[geo.first_row * geo.pitch + start..]; + int8_gemm_packed_strided(a, geo.pitch, &b.bands, &mut c[row0 * cols..], fs.len()); + report.tiles += fs.len() / TILE * b.bands.len(); + report.dp_steps += fs.len() / TILE * b.bands.len() * (k_pad / K_STEP); + Ok(()) +} diff --git a/crates/dr-isa-stream-lab/src/shape.rs b/crates/dr-isa-stream-lab/src/shape.rs index 02e5431a5..f450718e8 100644 --- a/crates/dr-isa-stream-lab/src/shape.rs +++ b/crates/dr-isa-stream-lab/src/shape.rs @@ -580,6 +580,15 @@ pub enum ReuseRefusal { /// output tile and every repeated run. Rows are the selected identities in /// order; rows past the selection are zero. pub struct PreparedB { + key: ReuseKey, + panels: Vec, +} + +/// What a prepared operand was prepared for, and the one rule for reusing it. +/// Shared by the BF16 [`PreparedB`] and the INT8 `PreparedI8B`, so the two +/// cannot drift apart. +#[derive(Debug, Clone)] +pub struct ReuseKey { /// The object it was packed for (fast path: one compare). object: u64, /// That object's selection, compared in full when the object differs: @@ -587,7 +596,46 @@ pub struct PreparedB { selection: Selection, frame: Frame, operand: OperandId, - panels: Vec, + cols: usize, +} + +impl ReuseKey { + /// The key of an operand prepared for `obj` from `b`. + pub fn of(obj: &VirtualObject, b: &impl BValues) -> ReuseKey { + ReuseKey { + object: obj.id, + selection: obj.selection.clone(), + frame: obj.frame().clone(), + operand: b.identity(), + cols: obj.cols, + } + } + + /// Whether the prepared operand may serve `obj` with operand `b`. + /// + /// The operand is compared by identity (source and revision), always. + /// For the object it was packed for, one identity compare covers the + /// selection, frame and width. For any other object the selection is + /// compared in full, then the frame issue and the width: a prepared B is + /// shared across objects only when all three are equal. + pub fn admit(&self, obj: &VirtualObject, b: &impl BValues) -> Result<(), ReuseRefusal> { + if self.operand != b.identity() { + return Err(ReuseRefusal::Operand); + } + if self.object == obj.id { + return Ok(()); + } + if self.selection != obj.selection { + return Err(ReuseRefusal::Selection); + } + if !self.frame.same_issue(obj.frame()) { + return Err(ReuseRefusal::Frame); + } + if self.cols != obj.cols { + return Err(ReuseRefusal::Columns); + } + Ok(()) + } } /// What preparing B did. @@ -625,10 +673,7 @@ impl PreparedB { let allocated = panels.iter().map(|p| p.data().len() * 2).sum(); Ok(( PreparedB { - object: obj.id, - selection: obj.selection.clone(), - frame: obj.frame().clone(), - operand: b.identity(), + key: ReuseKey::of(obj, b), panels, }, PrepReport { @@ -644,30 +689,10 @@ impl PreparedB { &self.panels[t] } - /// Whether this prepared B may serve `obj` with operand `b`. - /// - /// The operand is compared by identity (source and revision), always. - /// For the object it was packed for, one identity compare covers the - /// selection, frame and width. For any other object the selection is - /// compared in full, then the frame issue and the width: a prepared B is - /// shared across objects only when all three are equal. + /// Whether this prepared B may serve `obj` with operand `b` + /// ([`ReuseKey::admit`]). pub fn admit(&self, obj: &VirtualObject, b: &impl BValues) -> Result<(), ReuseRefusal> { - if self.operand != b.identity() { - return Err(ReuseRefusal::Operand); - } - if self.object == obj.id { - return Ok(()); - } - if self.selection != obj.selection { - return Err(ReuseRefusal::Selection); - } - if !self.frame.same_issue(obj.frame()) { - return Err(ReuseRefusal::Frame); - } - if self.panels.len() * TILE != obj.cols { - return Err(ReuseRefusal::Columns); - } - Ok(()) + self.key.admit(obj, b) } } diff --git a/crates/dr-isa-stream-lab/tests/integration/int8.rs b/crates/dr-isa-stream-lab/tests/integration/int8.rs index 783b7a705..02fe95f22 100644 --- a/crates/dr-isa-stream-lab/tests/integration/int8.rs +++ b/crates/dr-isa-stream-lab/tests/integration/int8.rs @@ -265,3 +265,97 @@ fn int8_refuses_a_b_table_narrower_than_the_object() { int8::staged(&obj, &l, &B, &mut Vec::new(), &mut Vec::new(), &mut want).unwrap(); assert_eq!(c, want); } + +/// The direct arm (A in place, B prepared once) is exact against the i64 +/// oracle and equal to the staged arm, at both sizes and with K tails; its +/// prepared B serves repeated runs unchanged. +#[test] +fn int8_direct_is_exact_and_reuses_its_prepared_b() { + use dr_isa_stream_lab::shape::BTable; + for dim in [64, 256] { + let fx = fx(4096, dim); + let lanes = U8Lanes::slab_in_order(&fx); + let socket = U8Socket::in_order(&lanes, dim); + let world = world(dim); + let bt = BTable::new(&B, 4096, dim); + for (start, len) in [(0, 63), (100, 65), (64, 128), (4096 - 512, 512)] { + let sel = Selection::region(SPACE_A, start, len); + let obj = VirtualObject::bind(&world, &socket, &fields(dim), dim, sel.clone()).unwrap(); + let l = int8::resolve(&obj, &world, &socket).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &bt).unwrap(); + let want: Vec = int8::oracle(&fx, &sel, &fields(dim), dim, &B) + .iter() + .map(|&v| i32::try_from(v).unwrap()) + .collect(); + let mut staged = vec![0i32; obj.c_len()]; + int8::staged(&obj, &l, &bt, &mut Vec::new(), &mut Vec::new(), &mut staged).unwrap(); + for run in 0..2 { + let mut c = vec![7i32; obj.c_len()]; + let r = int8::direct(&obj, &l, &prep, &bt, &mut c).unwrap(); + assert_eq!(c, want, "dim {dim} region ({start}, {len}) run {run}"); + assert_eq!(r.a_copied, 0); + assert_eq!(r.b_materialized, 0); + } + assert_eq!(staged, want); + } + } +} + +/// Lanes that are not one strided operand are refused, never copied: rows +/// out of order, a scattered mask, and a region whose padded read would +/// leave the lane's row. A prepared B for another operand is refused too. +#[test] +fn int8_direct_refuses_what_it_cannot_read_in_place() { + use dr_isa_stream_lab::gemm::DirectRefusal; + use dr_isa_stream_lab::shape::{BTable, ReuseRefusal}; + let dim = 64; + let fx = fx(4096, dim); + let lanes = U8Lanes::slab_in_order(&fx); + let world = world(dim); + let bt = BTable::new(&B, 4096, dim); + + let mut permuted = U8Socket::in_order(&lanes, dim); + permuted.rows.swap(3, 4); + let sel = Selection::region(SPACE_A, 0, 128); + let obj = VirtualObject::bind(&world, &permuted, &fields(dim), dim, sel).unwrap(); + let l = int8::resolve(&obj, &world, &permuted).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &bt).unwrap(); + let mut c = vec![0i32; obj.c_len()]; + assert!(matches!( + int8::direct(&obj, &l, &prep, &bt, &mut c), + Err(ShapeRefusal::Direct( + DirectRefusal::RowsNotConsecutive { .. } + )) + )); + + let socket = U8Socket::in_order(&lanes, dim); + let mask = Selection::mask(SPACE_A, 4096, (0..64).map(|k| k * 61 % 4096)); + let obj = VirtualObject::bind(&world, &socket, &fields(dim), dim, mask).unwrap(); + let l = int8::resolve(&obj, &world, &socket).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &bt).unwrap(); + assert!(matches!( + int8::direct(&obj, &l, &prep, &bt, &mut vec![0i32; obj.c_len()]), + Err(ShapeRefusal::NotARegion) + )); + + // The pitch is 4096 (already a multiple of 64). A 56-long region at + // 4040 pads to 64 and would read up to 4104, past its own row. + let sel = Selection::region(SPACE_A, 4040, 56); + let obj = VirtualObject::bind(&world, &socket, &fields(dim), dim, sel).unwrap(); + let l = int8::resolve(&obj, &world, &socket).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &bt).unwrap(); + assert!(matches!( + int8::direct(&obj, &l, &prep, &bt, &mut vec![0i32; obj.c_len()]), + Err(ShapeRefusal::OverRead { .. }) + )); + + let sel = Selection::region(SPACE_A, 0, 128); + let obj = VirtualObject::bind(&world, &socket, &fields(dim), dim, sel).unwrap(); + let l = int8::resolve(&obj, &world, &socket).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &bt).unwrap(); + let other = BTable::new(&B, 4096, dim); + assert!(matches!( + int8::direct(&obj, &l, &prep, &other, &mut vec![0i32; obj.c_len()]), + Err(ShapeRefusal::Reuse(ReuseRefusal::Operand)) + )); +} From d024630cc0e506cb4b4af776f0644cce5b7c61ad Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 17:51:54 +0000 Subject: [PATCH 2/9] dr-isa-stream-lab: record the INT8 direct arm results Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .../2026-11-10-int8-direct-packed-b.md | 21 +++++++++++++ .claude/board/entries/README.md | 3 +- crates/dr-isa-stream-lab/BENCH-SHAPE.md | 31 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-11-10-int8-direct-packed-b.md diff --git a/.claude/board/entries/2026-11-10-int8-direct-packed-b.md b/.claude/board/entries/2026-11-10-int8-direct-packed-b.md new file mode 100644 index 000000000..c6448a235 --- /dev/null +++ b/.claude/board/entries/2026-11-10-int8-direct-packed-b.md @@ -0,0 +1,21 @@ +# 2026-11-10 — INT8 direct arm on prepacked INT8 B + +**Status:** MEASURED (lab). Needs AdaWorldAPI/ndarray#418. + +**What.** `dr-isa-stream-lab`'s INT8 arm now reads A in place and takes B +from bands packed once (`PreparedI8B`, `PackedI8B`), through +`ndarray::simd::int8_gemm_packed_strided`. The prepared-operand reuse rule is +one `shape::ReuseKey` for BF16 and INT8. + +**Result** (one AMX VM, 41 warm runs, three runs). At 256 fields the GEMM +goes from 78.3–78.8 µs to 45.5–48.1 µs, end to end from 88.9–92.2 µs to +47.5–57.9 µs; at 64 fields from 3.4–3.6 µs to 0.98–1.01 µs. + +**Negative result.** One call per 16×16 tile (99–108 µs) was slower than +`gemm_i8` with its pack: per-call tile configuration and lost 2×2 blocking. +The pack only pays to hoist when the whole-matrix kernel takes it. + +**Governing question** (`dr-dt64-locality-execution-contract.md`): the VNNI +pack of B was recomputed per operation; it is now done once per binding and +reused under `ReuseKey` (operand identity and revision, selection, frame, +width). diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 7d336cb7f..be4331891 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -591 entries, 2026-08-06 .. 2026-11-10. +592 entries, 2026-08-06 .. 2026-11-10. | date | entry id | finding | file | |---|---|---|---| | 2026-11-10 | `lab-resolve-borrow-all` | | [2026-11-10-lab-resolve-borrow-all.md](2026-11-10-lab-resolve-borrow-all.md) | +| 2026-11-10 | `int8-direct-packed-b` | | [2026-11-10-int8-direct-packed-b.md](2026-11-10-int8-direct-packed-b.md) | | 2026-11-10 | `dt64-lane-borrow-all` | | [2026-11-10-dt64-lane-borrow-all.md](2026-11-10-dt64-lane-borrow-all.md) | | 2026-11-10 | `dt64-int8-facet-stride` | | [2026-11-10-dt64-int8-facet-stride.md](2026-11-10-dt64-int8-facet-stride.md) | | 2026-11-10 | `D-DT64-GEMM-SHAPE-1` | | [2026-11-10-dt64-gemm-shape.md](2026-11-10-dt64-gemm-shape.md) | diff --git a/crates/dr-isa-stream-lab/BENCH-SHAPE.md b/crates/dr-isa-stream-lab/BENCH-SHAPE.md index 247033661..480a80683 100644 --- a/crates/dr-isa-stream-lab/BENCH-SHAPE.md +++ b/crates/dr-isa-stream-lab/BENCH-SHAPE.md @@ -313,3 +313,34 @@ three are now fixed: The FACET STRIDE guard (`facet-stride-guard.sh --diff`) reports 0 hits on this change. ndarray is unchanged. + +## INT8 direct arm on prepacked INT8 B (ndarray#418) + +The OPTIMIZATION FOUND above, now built. `int8::direct` reads A in place at +the u8 lanes' pitch and takes B from `PreparedI8B` (one `PackedI8B` band per +16 output columns, packed once) through +`ndarray::simd::int8_gemm_packed_strided`. Reuse follows `ReuseKey`, the one +rule now shared with the BF16 `PreparedB`. + +One AMX VM, median of 41 warm runs, three runs +(`results/shape-2026-11-10-int8-direct-run*.txt`): + +| Row | 64 fields | 256 fields | +|---|---|---| +| `backend::gemm_i8` (packs B per call) | 3.4–3.6 µs | 78.3–78.8 µs | +| … of which band gather + VNNI pack | 1.9–2.1 µs | 28.6–29.4 µs | +| **direct GEMM (A in place, B prepared once)** | **0.98–1.01 µs** | **45.5–48.1 µs** (348–368 GMAC/s) | +| prepare B once | 6.4–6.7 µs | 105 µs | +| warm end to end, staged arm | 5.7–5.9 µs | 88.9–92.2 µs | +| **warm end to end, direct arm** | **2.9–3.0 µs** | **47.5–57.9 µs** | + +The direct arm allocates nothing and copies no A. Preparing B costs about +three `gemm_i8` packs, so it pays once B is reused three times. + +**A negative result kept on record.** The first version of this arm called +`int8_tile_gemm_16x16_packed_strided` once per 16×16 tile. It measured 99–108 +µs at 256 fields, slower than `gemm_i8` *with* its per-call pack: 256 calls +each pay the tile configuration (~105 ns), and the per-tile calls lose the +2×2 register blocking. Hoisting the pack was not enough; the whole-matrix +kernel had to take the prepacked bands. That is why ndarray#418 adds +`int8_gemm_packed_strided`. From ecb9d99715e4f140b08cb6a634e73a8f1e7e5f66 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:00:45 +0000 Subject: [PATCH 3/9] wip: G2L lab, one witnessed translation reused across GEMMs (untested) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/dr-isa-stream-lab/src/bin/shape.rs | 58 +++ crates/dr-isa-stream-lab/src/g2l.rs | 335 ++++++++++++++++++ crates/dr-isa-stream-lab/src/int8.rs | 43 +++ crates/dr-isa-stream-lab/src/lib.rs | 1 + crates/dr-isa-stream-lab/src/shape.rs | 14 + .../tests/integration/g2l.rs | 199 +++++++++++ .../tests/integration/main.rs | 1 + 7 files changed, 651 insertions(+) create mode 100644 crates/dr-isa-stream-lab/src/g2l.rs create mode 100644 crates/dr-isa-stream-lab/tests/integration/g2l.rs diff --git a/crates/dr-isa-stream-lab/src/bin/shape.rs b/crates/dr-isa-stream-lab/src/bin/shape.rs index 96feaf578..1954baf3c 100644 --- a/crates/dr-isa-stream-lab/src/bin/shape.rs +++ b/crates/dr-isa-stream-lab/src/bin/shape.rs @@ -526,6 +526,64 @@ fn main() { ); } + // G2L: a selection named in global ids, read on a source holding the + // population in its own row order. The translation is prepared once and + // reused; the anti-pattern re-translates on every operation. + { + use dr_isa_stream_lab::g2l::{global_of, translate, OrderedSource, TranslatedB}; + let dim = 64; + let fx = Fixture { + n: 4096, + props: dim, + seed: 0x6211, + }; + let world = World::with_extent(dim, "isa:f", Extent::DR64_CUBE); + let fields: Vec = (0..dim as u8).collect(); + let order: Vec = (0..4096).map(|l| (l * 1237 + 11) % 4096).collect(); + let source = OrderedSource::new(&fx, dim, order); + let socket = source.socket(dim); + let globals: Vec = (0..200).map(|k| global_of((k * 977 + 5) % 4096)).collect(); + let bt = BTable::new(&b, 4096, dim); + let tr = translate(&source, &globals).unwrap(); + let obj = VirtualObject::bind(&world, &socket, &fields, dim, tr.selection().clone()).unwrap(); + let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &tb).unwrap(); + let (mut a, mut c) = (Vec::new(), vec![0i32; obj.c_len()]); + let run = |a: &mut Vec, c: &mut [i32]| { + let l = int8::resolve(&obj, &world, &socket).unwrap(); + int8::staged_prepared(&obj, &l, &prep, &tb, a, c).unwrap(); + }; + run(&mut a, &mut c); + let want = c.clone(); + let lookups = source.lookups(); + let t_translate = median_ns(|| { + black_box(translate(&source, &globals).unwrap()); + }); + let t_prepare = median_ns(|| { + black_box(int8::PreparedI8B::prepare(&obj, &tb).unwrap()); + }); + let t_warm = median_ns(|| run(&mut a, &mut c)); + assert_eq!(c, want); + let t_per_op = median_ns(|| { + let tr = translate(&source, &globals).unwrap(); + let o = VirtualObject::bind(&world, &socket, &fields, dim, tr.selection().clone()).unwrap(); + let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); + let p = int8::PreparedI8B::prepare(&o, &tb).unwrap(); + let l = int8::resolve(&o, &world, &socket).unwrap(); + int8::staged_prepared(&o, &l, &p, &tb, &mut a, &mut c).unwrap(); + }); + assert_eq!(c, want); + println!( + "G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once {} ({} lookups) · prepare B once {} · warm run {} (0 lookups) · re-translate per operation {}\n", + fmt(t_translate), + tr.lookups(), + fmt(t_prepare), + fmt(t_warm), + fmt(t_per_op) + ); + let _ = lookups; + } + println!("| Metric | 64×64 | 256×256 |\n|---|---|---|"); for r in rows { println!("| {} | {} | {} |", r.metric, r.v[0], r.v[1]); diff --git a/crates/dr-isa-stream-lab/src/g2l.rs b/crates/dr-isa-stream-lab/src/g2l.rs new file mode 100644 index 000000000..88617a17a --- /dev/null +++ b/crates/dr-isa-stream-lab/src/g2l.rs @@ -0,0 +1,335 @@ +//! D-DT64-G2L-1: one witnessed global → local translation, bound once and +//! reused across repeated GEMMs. +//! +//! Two parties hold the same population in different row orders. The +//! consumer names its selection in **global ids**. The source holds its u8 +//! lanes in **its own order**, and DR's [`MapTable`] maps global id → +//! (tile, local position) for that order. Reading the source's lanes in the +//! consumer's order directly would be a gather, and is refused +//! ([`lance_graph_dt64::LaneRefusal::CoordinateSpace`]). +//! +//! ADMISSION B (#1640): preparation may establish coordinates; execution +//! operates on co-addressed sets; neither may hide an unaccounted gather. +//! So: +//! +//! 1. **Prepare, once, counted.** [`translate`] locates every selected global +//! id under one frame of the source's map table (every lookup is counted +//! by the table) and expresses the selection in the **source's** row order +//! ([`Selection`] in [`SPACE_B`]). It refuses a map whose coordinate +//! revision is not the source's current one (a stale mapping). +//! 2. **Bind as usual.** The object is bound on the source in the source's +//! own coordinate space, so DT64's ordinary checks apply and `borrow_all` +//! resolves it with one check. +//! 3. **Prepare B once.** The consumer's B is keyed by identity. It is read +//! through the translation ([`TranslatedB`]: position → global through the +//! map's pointer table, uncounted) only while packing, into the source's +//! row order. Its operand identity names the translation, so a prepared B +//! never outlives the translation it was built for. +//! 4. **Execute, many times, with zero lookups.** Every run reads the +//! source's lanes in place, in the source's order, against the prepared +//! B. The sum over K does not depend on K's order, so the result is the +//! consumer's answer. +//! +//! A source reorder advances its frames and its map table; the old binding +//! and everything prepared under it are then refused, and a fresh +//! translation is exact again. + +use std::cell::Cell; +use std::sync::atomic::{AtomicU64, Ordering}; + +use lance_graph_dt64::dr::{ + Frame, FrameRevisions, LocalMap, LocateRefusal, MapTable, TileId, Width, +}; +use lance_graph_dt64::CoordSpace; + +use crate::bind::READER; +use crate::fixture::Fixture; +use crate::int8::{a_u8, U8Lanes, U8Socket}; +use crate::shape::{BValues, InnerSource, OperandId, OperandSource, Selection}; + +/// The source's own row order of the population. +pub const SPACE_B: CoordSpace = CoordSpace { + population: 1, + coordinates: 3, +}; + +/// Positions per DR256 tile. +const TILE_POSITIONS: usize = 256; + +/// The global id of canonical identity `i`: neither its index nor any +/// source's position. +pub fn global_of(i: usize) -> u64 { + 1_000 + 7 * i as u64 +} + +/// The canonical identity of a global id. +pub fn identity_of(g: u64) -> usize { + ((g - 1_000) / 7) as usize +} + +/// A source holding the population's u8 lanes in its own row order, with +/// the DR map table for that order. Position `l` holds identity `order[l]`. +pub struct OrderedSource { + lanes: U8Lanes, + map: MapTable, + tiles: Vec, + order: Vec, +} + +impl OrderedSource { + /// `props` lanes over `fx.n` identities in `order` (a permutation). + pub fn new(fx: &Fixture, props: usize, order: Vec) -> OrderedSource { + assert_eq!(order.len(), fx.n); + let lanes = U8Lanes::from_fn(fx.n, props, |l, p| a_u8(fx, order[l], p)); + let tiles: Vec = (0..fx.n.div_ceil(TILE_POSITIONS) as u32).collect(); + let mut src = OrderedSource { + lanes, + map: MapTable::new(1, 1), + tiles, + order, + }; + src.publish(); + src + } + + /// Reorder the source in place: rewrite its lanes, advance its map table + /// and then its frames to the map's new coordinate revision. + pub fn reorder(&mut self, fx: &Fixture, order: Vec) { + assert_eq!(order.len(), fx.n); + self.order = order; + let order = &self.order; + self.lanes.rewrite(|l, p| a_u8(fx, order[l], p)); + self.publish(); + } + + /// Reorder the lanes but NOT the map table: the stale-mapping case. + pub fn reorder_lanes_only(&mut self, fx: &Fixture, order: Vec) { + self.order = order; + let order = &self.order; + self.lanes.rewrite(|l, p| a_u8(fx, order[l], p)); + let rev = self.lanes.frames_mut().revisions(); + self.lanes.frames_mut().advance(FrameRevisions { + coordinates: rev.coordinates.wrapping_add(1), + ..rev + }); + } + + /// Replace every tile's map with the current order, then advance the + /// source's frames to the map's coordinate revision. + fn publish(&mut self) { + let changes = self + .tiles + .iter() + .map(|&t| { + let lo = t as usize * TILE_POSITIONS; + let hi = (lo + TILE_POSITIONS).min(self.order.len()); + let globals: Vec = self.order[lo..hi].iter().map(|&i| global_of(i)).collect(); + (t, Some(LocalMap::prepare(Width::W256, &globals).expect("a tile holds 256 ids"))) + }) + .collect(); + self.map.advance(1, 1, changes); + let coordinates = self.map.revisions().coordinates; + let rev = self.lanes.frames_mut().revisions(); + self.lanes + .frames_mut() + .advance(FrameRevisions { coordinates, ..rev }); + } + + /// The DT64 socket over this source's lanes, in [`SPACE_B`]. + pub fn socket(&self, props: usize) -> U8Socket<'_> { + U8Socket { + lanes: &self.lanes, + rows: (0..props).collect(), + space: SPACE_B, + scope: READER, + borrows: Cell::new(0), + } + } + + /// Global → local lookups the map table has served, over every tile. + pub fn lookups(&self) -> u64 { + let frame = self.map.seal(); + self.tiles + .iter() + .filter_map(|&t| self.map.map(&frame, t).ok()) + .map(|m| m.lookups()) + .sum() + } + + /// Identities held by the source. + pub fn len(&self) -> usize { + self.order.len() + } + + /// True if the source holds no identity. + pub fn is_empty(&self) -> bool { + self.order.is_empty() + } +} + +/// Why a translation was refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum G2lRefusal { + /// The map table's coordinate revision is not the source's: the source's + /// order moved without its map (a stale mapping). + MapNotSource { map: u64, source: u64 }, + /// A selected global id is not held by the source, or is held twice. + Locate { global: u64, refusal: LocateRefusal }, + /// A global id is selected twice. + Duplicate { global: u64 }, + /// The translation was made under another order of the source. + Stale { made: u64, now: u64 }, +} + +static NEXT_TRANSLATION: AtomicU64 = AtomicU64::new(1); + +/// A witnessed translation of a global-id selection into the source's row +/// order, made once under one frame of the source's map table. +#[derive(Debug, Clone)] +pub struct Translation { + id: u64, + /// The source's coordinate revision the translation was made under. + coordinates: u64, + frame: Frame, + tiles: Vec, + selection: Selection, + lookups: u64, +} + +impl Translation { + /// The selection in the source's row order ([`SPACE_B`]). + pub fn selection(&self) -> &Selection { + &self.selection + } + + /// Lookups this translation cost (once, at preparation). + pub fn lookups(&self) -> u64 { + self.lookups + } + + /// Refuses once the source's order has moved since the translation was + /// made. One compare. + pub fn check(&self, source: &OrderedSource) -> Result<(), G2lRefusal> { + let now = source.lanes.frames().revisions().coordinates; + if now != self.coordinates { + return Err(G2lRefusal::Stale { + made: self.coordinates, + now, + }); + } + Ok(()) + } +} + +/// Prepare: locate every global id under one frame of the source's map +/// table and express the selection in the source's own row order. Counted; +/// done once. +pub fn translate(source: &OrderedSource, globals: &[u64]) -> Result { + let map = source.map.revisions().coordinates; + let src = source.lanes.frames().revisions().coordinates; + if map != src { + return Err(G2lRefusal::MapNotSource { map, source: src }); + } + let before = source.lookups(); + let frame = source.map.seal(); + let mut positions = Vec::with_capacity(globals.len()); + for &g in globals { + let (ordinal, local) = source + .map + .locate(&frame, &source.tiles, g) + .map_err(|refusal| G2lRefusal::Locate { global: g, refusal })?; + positions.push(ordinal as usize * TILE_POSITIONS + usize::from(local)); + } + let mut sorted = positions.clone(); + sorted.sort_unstable(); + if let Some(w) = sorted.windows(2).find(|w| w[0] == w[1]) { + let local = w[0]; + let global = globals[positions.iter().position(|&p| p == local).unwrap_or(0)]; + return Err(G2lRefusal::Duplicate { global }); + } + let selection = Selection::mask(SPACE_B, source.len(), positions); + Ok(Translation { + id: NEXT_TRANSLATION.fetch_add(1, Ordering::Relaxed), + coordinates: src, + frame, + tiles: source.tiles.clone(), + selection, + lookups: source.lookups() - before, + }) +} + +/// The consumer's B (keyed by canonical identity) read at the source's row +/// positions through a translation. Used only to prepare B; counts its reads +/// so a test can show execution never comes back to it. +pub struct TranslatedB<'a, T> { + inner: &'a T, + inner_source: InnerSource, + source: &'a OrderedSource, + translation: &'a Translation, + reads: Cell, +} + +impl<'a, T: BValues> TranslatedB<'a, T> { + /// Refuses a translation made under another order of the source. + pub fn new( + inner: &'a T, + source: &'a OrderedSource, + translation: &'a Translation, + ) -> Result { + translation.check(source)?; + let inner_source = match inner.identity().source { + OperandSource::Function { seed } => InnerSource::Function { seed }, + OperandSource::Table { instance } => InnerSource::Table { instance }, + OperandSource::Translated { .. } => panic!("a translation of a translation"), + }; + Ok(TranslatedB { + inner, + inner_source, + source, + translation, + reads: Cell::new(0), + }) + } + + /// Values read through the translation so far. + pub fn reads(&self) -> u64 { + self.reads.get() + } + + /// Source position → canonical identity, through the map's pointer table + /// at the translation's frame (uncounted: the inverse read). + fn identity_at(&self, position: usize) -> usize { + self.reads.set(self.reads.get() + 1); + let tile = self.translation.tiles[position / TILE_POSITIONS]; + let local = u8::try_from(position % TILE_POSITIONS).expect("a tile holds 256 positions"); + let map = self + .source + .map + .map(&self.translation.frame, tile) + .expect("the translation's own frame reads its own table"); + identity_of(map.global(local).expect("a translated position is mapped")) + } +} + +impl BValues for TranslatedB<'_, T> { + fn identity(&self) -> OperandId { + OperandId { + source: OperandSource::Translated { + translation: self.translation.id, + inner: self.inner_source, + }, + rev: self.inner.identity().rev, + } + } + fn bf16_at(&self, i: usize, j: usize) -> u16 { + self.inner.bf16_at(self.identity_at(i), j) + } + fn i8_at(&self, i: usize, j: usize) -> i8 { + self.inner.i8_at(self.identity_at(i), j) + } + fn extent(&self) -> Option<(usize, usize)> { + self.inner + .extent() + .map(|(_, cols)| (self.source.len(), cols)) + } +} diff --git a/crates/dr-isa-stream-lab/src/int8.rs b/crates/dr-isa-stream-lab/src/int8.rs index 39c0e8444..6c826dd24 100644 --- a/crates/dr-isa-stream-lab/src/int8.rs +++ b/crates/dr-isa-stream-lab/src/int8.rs @@ -102,11 +102,27 @@ impl U8Lanes { } } + /// The frame owner. + pub fn frames(&self) -> &FrameStore { + &self.frames + } + /// Mutable frame owner, for advancing the source after a change. pub fn frames_mut(&mut self) -> &mut FrameStore { &mut self.frames } + /// Rewrite every value in place (`value(i, p)` for identity row `i` of + /// lane `p`). The caller advances the frames afterwards. + pub fn rewrite(&mut self, value: impl Fn(usize, usize) -> u8) { + let (pitch, props) = (self.pitch, self.data.len() / self.pitch.max(1)); + for p in 0..props { + for i in 0..self.n { + self.data[p * pitch + i] = value(i, p); + } + } + } + /// Physical row `r`, borrowed in place. pub fn row(&self, r: usize) -> Option> { (r * self.pitch + self.n <= self.data.len()).then_some(U8Lane { @@ -528,3 +544,30 @@ fn run_rows( report.dp_steps += fs.len() / TILE * b.bands.len() * (k_pad / K_STEP); Ok(()) } + +/// The staged arm with a prepared B: A copied (one region copy per lane, or a +/// per-value walk for a mask) into `a`, B taken from the prepared bands. For +/// selections the direct arm cannot read in place, such as a scattered mask. +/// `c` is overwritten. +pub fn staged_prepared( + obj: &VirtualObject, + lanes: &U8Lanes64<'_>, + b: &PreparedI8B, + operand: &impl BValues, + a: &mut Vec, + c: &mut [i32], +) -> Result { + b.key.admit(obj, operand).map_err(ShapeRefusal::Reuse)?; + let a_copied = stage_a(obj, lanes, a)?; + let k_pad = obj.selection().len().div_ceil(K_STEP) * K_STEP; + let rows = obj.fields().len(); + assert_eq!(c.len(), rows * obj.cols()); + int8_gemm_packed_strided(a, k_pad, &b.bands, c, rows); + Ok(Int8Report { + k_pad, + a_copied, + b_materialized: 0, + tiles: rows / TILE * b.bands.len(), + dp_steps: rows / TILE * b.bands.len() * (k_pad / K_STEP), + }) +} diff --git a/crates/dr-isa-stream-lab/src/lib.rs b/crates/dr-isa-stream-lab/src/lib.rs index e9ef47538..b31b7646b 100644 --- a/crates/dr-isa-stream-lab/src/lib.rs +++ b/crates/dr-isa-stream-lab/src/lib.rs @@ -22,6 +22,7 @@ pub mod bind; pub mod cube; pub mod fixture; +pub mod g2l; pub mod gemm; pub mod int8; pub mod oracle; diff --git a/crates/dr-isa-stream-lab/src/shape.rs b/crates/dr-isa-stream-lab/src/shape.rs index f450718e8..6e0ac63c8 100644 --- a/crates/dr-isa-stream-lab/src/shape.rs +++ b/crates/dr-isa-stream-lab/src/shape.rs @@ -476,6 +476,20 @@ pub enum OperandSource { Function { seed: u64 }, /// An immutable table, by the construction that filled it. Table { instance: u64 }, + /// Another operand read through a witnessed global -> local translation + /// (`crate::g2l`): the translation's instance and the inner operand's + /// source. Two translations of one operand are two operands. + Translated { + translation: u64, + inner: InnerSource, + }, +} + +/// The source of an operand read through a translation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InnerSource { + Function { seed: u64 }, + Table { instance: u64 }, } impl BValues for BOperand { diff --git a/crates/dr-isa-stream-lab/tests/integration/g2l.rs b/crates/dr-isa-stream-lab/tests/integration/g2l.rs new file mode 100644 index 000000000..ac75fad8d --- /dev/null +++ b/crates/dr-isa-stream-lab/tests/integration/g2l.rs @@ -0,0 +1,199 @@ +//! D-DT64-G2L-1: one witnessed global -> local translation, made once and +//! reused across repeated GEMMs, against an oracle computed in global ids +//! alone. + +use dr_isa_stream_lab::bind::{World, SPACE_A}; +use dr_isa_stream_lab::fixture::Fixture; +use dr_isa_stream_lab::g2l::{global_of, identity_of, translate, G2lRefusal, OrderedSource, TranslatedB}; +use dr_isa_stream_lab::int8::{self, a_u8, b_i8}; +use dr_isa_stream_lab::shape::{BOperand, BTable, ReuseRefusal, Selection, ShapeRefusal, VirtualObject}; +use lance_graph_dt64::dr::{FrameRefusal, LocateRefusal}; +use lance_graph_dt64::{Extent, LaneRefusal}; + +const B: BOperand = BOperand { + seed: 0xB0B, + rev: 1, +}; +const N: usize = 4096; +const DIM: usize = 64; + +fn world() -> World { + World::with_extent(DIM, "isa:f", Extent::DR64_CUBE) +} + +fn fields() -> Vec { + (0..DIM).map(|f| f as u8).collect() +} + +fn fx() -> Fixture { + Fixture { + n: N, + props: DIM, + seed: 0x6211, + } +} + +/// A permutation of `0..N` (`mul` odd, so a bijection mod 2^12). +fn order(mul: usize, add: usize) -> Vec { + (0..N).map(|l| (l * mul + add) % N).collect() +} + +/// The consumer's selection, in global ids, in the consumer's own order. +fn wanted() -> Vec { + (0..200).map(|k| global_of((k * 977 + 5) % N)).collect() +} + +/// `C[p][j] = sum over the selected global ids of A(g, p) * B(g, j)`, +/// computed from the fixture and B by identity: no map, no source order. +fn oracle(fx: &Fixture, globals: &[u64]) -> Vec { + let mut c = vec![0i64; DIM * DIM]; + for &g in globals { + let i = identity_of(g); + for p in 0..DIM { + let a = i64::from(a_u8(fx, i, p)); + for j in 0..DIM { + c[p * DIM + j] += a * i64::from(b_i8(&B, i, j)); + } + } + } + c.iter().map(|&v| i32::try_from(v).unwrap()).collect() +} + +#[test] +fn a_translation_is_exact_and_reused_with_zero_further_lookups() { + let fx = fx(); + let source = OrderedSource::new(&fx, DIM, order(1237, 11)); + let socket = source.socket(DIM); + let world = world(); + let globals = wanted(); + let want = oracle(&fx, &globals); + + let tr = translate(&source, &globals).unwrap(); + // Every id is located once across the 16 tiles of the population. + assert_eq!(tr.lookups(), (globals.len() * N / 256) as u64); + let obj = VirtualObject::bind(&world, &socket, &fields(), DIM, tr.selection().clone()).unwrap(); + let bt = BTable::new(&B, N, DIM); + let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &tb).unwrap(); + let (lookups, reads) = (source.lookups(), tb.reads()); + assert!(reads > 0); + + let (mut a, mut c) = (Vec::new(), vec![0i32; obj.c_len()]); + for run in 0..5 { + let lanes = int8::resolve(&obj, &world, &socket).unwrap(); + int8::staged_prepared(&obj, &lanes, &prep, &tb, &mut a, &mut c).unwrap(); + assert_eq!(c, want, "run {run}"); + } + assert_eq!(source.lookups(), lookups, "execution never looks an id up"); + assert_eq!(tb.reads(), reads, "execution never reads B through the map"); +} + +/// Without a translation, the consumer's order cannot be read on the source. +#[test] +fn the_untranslated_cross_space_read_stays_refused() { + let fx = fx(); + let source = OrderedSource::new(&fx, DIM, order(1237, 11)); + let socket = source.socket(DIM); + let ids = wanted().iter().map(|&g| identity_of(g)).collect::>(); + let sel = Selection::mask(SPACE_A, N, ids); + let err = VirtualObject::bind(&world(), &socket, &fields(), DIM, sel).unwrap_err(); + assert!( + matches!(err, ShapeRefusal::Lane(LaneRefusal::CoordinateSpace { .. })), + "{err:?}" + ); +} + +#[test] +fn a_reorder_refuses_everything_prepared_under_the_old_order() { + let fx = fx(); + let mut source = OrderedSource::new(&fx, DIM, order(1237, 11)); + let world = world(); + let globals = wanted(); + let bt = BTable::new(&B, N, DIM); + let (tr, obj, prep) = { + let socket = source.socket(DIM); + let tr = translate(&source, &globals).unwrap(); + let obj = VirtualObject::bind(&world, &socket, &fields(), DIM, tr.selection().clone()).unwrap(); + let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &tb).unwrap(); + (tr, obj, prep) + }; + + source.reorder(&fx, order(3001, 7)); + let socket = source.socket(DIM); + assert!(matches!( + int8::resolve(&obj, &world, &socket), + Err(ShapeRefusal::Lane(LaneRefusal::Frame(FrameRefusal::CoordinatesChanged))) + )); + assert!(matches!(tr.check(&source), Err(G2lRefusal::Stale { .. }))); + assert!(matches!( + TranslatedB::new(&bt, &source, &tr), + Err(G2lRefusal::Stale { .. }) + )); + drop(prep); + + // A fresh translation under the new order is exact again. + let tr2 = translate(&source, &globals).unwrap(); + let obj2 = VirtualObject::bind(&world, &socket, &fields(), DIM, tr2.selection().clone()).unwrap(); + let tb2 = TranslatedB::new(&bt, &source, &tr2).unwrap(); + let prep2 = int8::PreparedI8B::prepare(&obj2, &tb2).unwrap(); + let lanes = int8::resolve(&obj2, &world, &socket).unwrap(); + let mut c = vec![0i32; obj2.c_len()]; + int8::staged_prepared(&obj2, &lanes, &prep2, &tb2, &mut Vec::new(), &mut c).unwrap(); + assert_eq!(c, oracle(&fx, &globals)); +} + +/// The source reordered its lanes without its map: the map no longer +/// describes the source, so no translation is made from it. +#[test] +fn a_stale_mapping_is_refused() { + let fx = fx(); + let mut source = OrderedSource::new(&fx, DIM, order(1237, 11)); + source.reorder_lanes_only(&fx, order(3001, 7)); + assert!(matches!( + translate(&source, &wanted()), + Err(G2lRefusal::MapNotSource { .. }) + )); +} + +#[test] +fn unknown_and_repeated_ids_are_refused() { + let fx = fx(); + let source = OrderedSource::new(&fx, DIM, order(1237, 11)); + assert!(matches!( + translate(&source, &[global_of(3), 5]), + Err(G2lRefusal::Locate { + global: 5, + refusal: LocateRefusal::Unmapped + }) + )); + assert!(matches!( + translate(&source, &[global_of(3), global_of(9), global_of(3)]), + Err(G2lRefusal::Duplicate { .. }) + )); +} + +/// Two translations of the same selection are two operands: B prepared +/// through one is refused through the other. +#[test] +fn a_prepared_b_belongs_to_its_translation() { + let fx = fx(); + let source = OrderedSource::new(&fx, DIM, order(1237, 11)); + let socket = source.socket(DIM); + let world = world(); + let bt = BTable::new(&B, N, DIM); + let tr1 = translate(&source, &wanted()).unwrap(); + let tr2 = translate(&source, &wanted()).unwrap(); + let obj = VirtualObject::bind(&world, &socket, &fields(), DIM, tr1.selection().clone()).unwrap(); + let tb1 = TranslatedB::new(&bt, &source, &tr1).unwrap(); + let tb2 = TranslatedB::new(&bt, &source, &tr2).unwrap(); + let prep = int8::PreparedI8B::prepare(&obj, &tb1).unwrap(); + let lanes = int8::resolve(&obj, &world, &socket).unwrap(); + let mut c = vec![0i32; obj.c_len()]; + assert!(matches!( + int8::staged_prepared(&obj, &lanes, &prep, &tb2, &mut Vec::new(), &mut c), + Err(ShapeRefusal::Reuse(ReuseRefusal::Operand)) + )); + int8::staged_prepared(&obj, &lanes, &prep, &tb1, &mut Vec::new(), &mut c).unwrap(); + assert_eq!(c, oracle(&fx, &wanted())); +} diff --git a/crates/dr-isa-stream-lab/tests/integration/main.rs b/crates/dr-isa-stream-lab/tests/integration/main.rs index c8377bb6d..80bb1dedd 100644 --- a/crates/dr-isa-stream-lab/tests/integration/main.rs +++ b/crates/dr-isa-stream-lab/tests/integration/main.rs @@ -1,3 +1,4 @@ +pub mod g2l; pub mod geometry; pub mod int8; pub mod numeric; From f97bba5edef1e0ca35f432fdbc61912c4ecf863d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:09:51 +0000 Subject: [PATCH 4/9] dr-isa-stream-lab: time the B pack through ndarray::simd, not hpc The benchmark's pack-timing row called ndarray::hpc::amx_matmul::vnni_pack_i8 directly. hpc is reached only through the ndarray::simd polyfill, so the row now times PackedI8B::pack_from, which packs into the same VNNI layout. The measured share is unchanged (23.1 us, 39% of the gemm_i8 row at 256 fields). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/dr-isa-stream-lab/src/bin/shape.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/dr-isa-stream-lab/src/bin/shape.rs b/crates/dr-isa-stream-lab/src/bin/shape.rs index 96feaf578..d9cbc388d 100644 --- a/crates/dr-isa-stream-lab/src/bin/shape.rs +++ b/crates/dr-isa-stream-lab/src/bin/shape.rs @@ -376,18 +376,19 @@ fn main() { assert_eq!(c8, want8); // The B packing gemm_i8 does inside every call, timed on its own: // per 16-column band, gather the band (K x 16) and VNNI-pack it with - // ndarray's own `vnni_pack_i8` (int8_gemm_amx_tiled_rb's steps, minus - // its three buffer allocations, which the GEMM row reports). + // `ndarray::simd::PackedI8B::pack_from` (the same layout and loop + // int8_gemm_amx_tiled_rb packs with, minus its three buffer + // allocations, which the GEMM row reports). let k8 = r8.k_pad; assert_eq!(b8.len(), k8 * dim, "B materialized row-major"); - let (mut band, mut packed) = (vec![0i8; k8 * 16], vec![0i8; k8 * 16]); + let (mut band, mut packed) = (vec![0i8; k8 * 16], ndarray::simd::PackedI8B::zeroed(k8)); let t8_pack = median_ns(|| { for j in (0..dim).step_by(16) { for kk in 0..k8 { band[kk * 16..kk * 16 + 16] .copy_from_slice(&b8[kk * dim + j..kk * dim + j + 16]); } - ndarray::hpc::amx_matmul::vnni_pack_i8(&band, &mut packed, k8, 16); + packed.pack_from(&band); black_box(&packed); } }); From 041ca3a40561084b9ac1727eaa80217ba3793273 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:18:31 +0000 Subject: [PATCH 5/9] =?UTF-8?q?feat(dr-isa-stream-lab):=20G2L=20lab=20fals?= =?UTF-8?q?ifier=20=E2=80=94=20one=20witnessed=20translation=20reused=20ac?= =?UTF-8?q?ross=20GEMMs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D-DT64-G2L-1. A selection named in global ids is translated once under one frame of the source's DR MapTable into the source's own row order (counted lookups). The object is bound in the source's space, B is prepared once through the translation (one map read per row), and every run reads the source's lanes in place with zero lookups. Cross-space reads without a translation stay refused (ADMISSION B, #1640). One AMX VM, three runs: translate once 68-85 us (3200 lookups), prepare B 37-47 us, warm run 16-22 us with 0 lookups; re-translating per operation 128-166 us (about 8x). Six falsifiers in tests/integration/g2l.rs, four disable runs red. Contract doc G2L row moves from OPEN to measured (lab). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .../board/entries/2026-11-10-dt64-g2l-lab.md | 27 ++++++++++ .claude/board/entries/README.md | 3 +- .../dr-dt64-locality-execution-contract.md | 2 +- crates/dr-isa-stream-lab/BENCH-SHAPE.md | 53 +++++++++++++++++++ .../results/shape-2026-11-10-g2l-run1.txt | 35 ++++++++++++ .../results/shape-2026-11-10-g2l-run2.txt | 35 ++++++++++++ .../results/shape-2026-11-10-g2l-run3.txt | 35 ++++++++++++ crates/dr-isa-stream-lab/src/bin/shape.rs | 15 +++--- crates/dr-isa-stream-lab/src/g2l.rs | 20 +++++-- .../tests/integration/g2l.rs | 24 ++++++--- 10 files changed, 231 insertions(+), 18 deletions(-) create mode 100644 .claude/board/entries/2026-11-10-dt64-g2l-lab.md create mode 100644 crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt create mode 100644 crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt create mode 100644 crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt diff --git a/.claude/board/entries/2026-11-10-dt64-g2l-lab.md b/.claude/board/entries/2026-11-10-dt64-g2l-lab.md new file mode 100644 index 000000000..9d68f3f35 --- /dev/null +++ b/.claude/board/entries/2026-11-10-dt64-g2l-lab.md @@ -0,0 +1,27 @@ +# 2026-11-10 — D-DT64-G2L-1: one witnessed translation reused across GEMMs + +**Status:** MEASURED (lab). Closes the G2L row of +`dr-dt64-locality-execution-contract.md` from OPEN to measured in the lab. +Stacked on #1992 (needs ndarray#418). + +**What.** A selection named in global ids is translated once, under one frame +of the source's DR `MapTable`, into the source's own row order (counted +lookups). The object is bound in the source's space; B is prepared once +through the translation; every run then reads the source's lanes in place +with zero lookups. Cross-space reads without a translation stay refused, and +the translation does not grant Gather (ADMISSION B #1640). + +**Result** (one AMX VM, three runs): translate once 68–85 µs (3200 lookups), +prepare B once 37–47 µs, warm run 16–22 µs with 0 lookups; re-translating per +operation 128–166 µs (about 8×). + +**Falsifiers:** oracle in global ids only; reorder refuses binding, +translation and operand; stale mapping refused; prepared B bound to its +translation; unknown and repeated ids refused. Four disable runs, each red. + +**Governing question:** the location of each selected id is resolved once +per translation; B's per-row identity once per preparation; per operation, +nothing. + +**Trap:** a disable run leaves its build behind. The bench reported 254 µs +for preparing B from the last disable run's binary; rebuilt, 37–47 µs. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index be4331891..215509f8f 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -592 entries, 2026-08-06 .. 2026-11-10. +593 entries, 2026-08-06 .. 2026-11-10. | date | entry id | finding | file | |---|---|---|---| @@ -34,6 +34,7 @@ exactly one of them, never both. | 2026-11-10 | `dt64-lane-borrow-all` | | [2026-11-10-dt64-lane-borrow-all.md](2026-11-10-dt64-lane-borrow-all.md) | | 2026-11-10 | `dt64-int8-facet-stride` | | [2026-11-10-dt64-int8-facet-stride.md](2026-11-10-dt64-int8-facet-stride.md) | | 2026-11-10 | `D-DT64-GEMM-SHAPE-1` | | [2026-11-10-dt64-gemm-shape.md](2026-11-10-dt64-gemm-shape.md) | +| 2026-11-10 | `D-DT64-G2L-1` | | [2026-11-10-dt64-g2l-lab.md](2026-11-10-dt64-g2l-lab.md) | | 2026-11-08 | `dr-isa-stream-lab` | | [2026-11-08-dr-isa-stream-lab.md](2026-11-08-dr-isa-stream-lab.md) | | 2026-11-07 | `D-DT64-7-L5b` | | [2026-11-07-dt64-l5b-historical-basis.md](2026-11-07-dt64-l5b-historical-basis.md) | | 2026-11-07 | `D-DT64-7-L5a` | | [2026-11-07-dt64-l5a-exact-basis-witness.md](2026-11-07-dt64-l5a-exact-basis-witness.md) | diff --git a/.claude/knowledge/dr-dt64-locality-execution-contract.md b/.claude/knowledge/dr-dt64-locality-execution-contract.md index 1f5048e74..d1ca70adf 100644 --- a/.claude/knowledge/dr-dt64-locality-execution-contract.md +++ b/.claude/knowledge/dr-dt64-locality-execution-contract.md @@ -122,7 +122,7 @@ field mapping, operand identity and version. | Lab resolve on `borrow_all` (follow-up) | MEASURED, not merged | Resolve at 256 fields 1.6–2.0 µs vs 13.8–13.9 µs (#1987 as first measured) and 26–28 µs (`main` after the provenance fix); the old cost was `LaneBinding::addr`, a linear scan called per field | | INT8 vs BF16 over the same object (follow-up) | MEASURED, not merged | `u8 × i8 → i32` end to end 66–101 µs vs BF16 103–160 µs at 256 fields on one VM; INT8 still copies A (5–7 µs) and re-packs B per call (23–29 µs, 36–38% of its GEMM), because ndarray has no strided-A / prepacked-B INT8 entry | | #1987's "B preparation ≈ GEMM" | REJECTED-BY-MEASUREMENT | That timer included the synthetic operand's hash; from a table built once, prepared B costs about a third of the GEMM | -| G2L across different source orders | OPEN | Not demonstrated: every fixture shared one coordinate space, and differing spaces were refused, not translated | +| G2L across different source orders (D-DT64-G2L-1) | MEASURED, lab | One translation under one `MapTable` frame (counted lookups), reused across runs with 0 lookups; warm run 16–22 µs vs 128–166 µs re-translating per operation; reorder, stale mapping and foreign translation refused. Still OPEN for production: who owns the translation, and frames across a restore (#1993) | | Foundry operational-ontology parity | HYPOTHESIS | Research goal; nothing here establishes it | All timings: one VM (Xeon with AMX-BF16, 4 vCPU), single-threaded, medians of diff --git a/crates/dr-isa-stream-lab/BENCH-SHAPE.md b/crates/dr-isa-stream-lab/BENCH-SHAPE.md index 480a80683..1d5483ce0 100644 --- a/crates/dr-isa-stream-lab/BENCH-SHAPE.md +++ b/crates/dr-isa-stream-lab/BENCH-SHAPE.md @@ -344,3 +344,56 @@ each pay the tile configuration (~105 ns), and the per-tile calls lose the 2×2 register blocking. Hoisting the pack was not enough; the whole-matrix kernel had to take the prepacked bands. That is why ndarray#418 adds `int8_gemm_packed_strided`. + +## D-DT64-G2L-1: one witnessed translation, reused (`src/g2l.rs`) + +The consumer names its selection in **global ids**. The source holds its u8 +lanes in **its own row order** (`SPACE_B`), described by a DR `MapTable` (16 +DR256 tiles over 4096 identities). Reading the source's lanes in the +consumer's order is a gather and stays refused (`CoordinateSpace`). + +ADMISSION B (#1640): preparation may establish coordinates; execution operates +on co-addressed sets. So: + +1. `translate` locates every selected global id once, under one frame of the + map table (counted lookups), and returns the selection **in the source's + row order**. It refuses a map whose coordinate revision is not the source's + (a stale mapping). +2. The object is bound on the source, in the source's space, as usual. +3. B is prepared once through `TranslatedB` (position → global via the map's + pointer table, one read per row); its operand identity names the + translation. +4. Every run reads the source's lanes in place, in the source's order, against + the prepared B, with **zero** lookups. A sum over K does not depend on K's + order, so the result is the consumer's answer. + +One AMX VM, three runs (`results/shape-2026-11-10-g2l-run*.txt`), 64 fields × +64 columns, 200 global ids: + +| Step | Time | Lookups | +|---|---|---| +| translate once | 68–85 µs | 3200 (200 ids × 16 tiles) | +| prepare B once | 37–47 µs | 200 map reads | +| **warm run** (resolve + stage A + GEMM) | **16–22 µs** | **0** | +| re-translate on every operation (the anti-pattern) | 128–166 µs | 3200 + 200 | + +Re-translating per operation costs about 8× the warm run. The first version of +`TranslatedB` resolved position → identity once per (row, column) and took +241–255 µs to prepare B: 64× the map reads needed. One read per row fixed it +(test-pinned: exactly 200 reads). + +Falsifiers (`tests/integration/g2l.rs`), each disable-verified red: + +| Test | Disable that turns it red | +|---|---| +| exact vs an oracle computed in global ids alone; 5 runs, 0 further lookups, 0 further map reads | per-row cache removed | +| a source reorder refuses the old binding (`CoordinatesChanged`), the translation and the operand; a fresh translation is exact | staleness check in `TranslatedB::new` removed | +| a stale mapping (lanes reordered, map not) is refused | map-vs-source coordinate check removed | +| a prepared B from one translation is refused under another | translated identity ignoring the translation | +| the untranslated cross-space read stays refused | (DT64's existing check) | +| unknown and repeated ids are refused | — | + +A measurement trap worth keeping: after the disable runs, the benchmark binary +was the one built by the last disable run (cache off), and reported 254 µs +for preparing B. Rebuilding from the committed source gave 37–47 µs. A disable +run leaves its build behind; re-measure only after a rebuild. diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt new file mode 100644 index 000000000..b6e92b876 --- /dev/null +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt @@ -0,0 +1,35 @@ +Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K +Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). + +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 380.1 µs + +G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once 84.6 µs (3200 lookups) · prepare B once 47.3 µs · warm run 21.5 µs (0 lookups) · re-translate per operation 165.6 µs + +| Metric | 64×64 | 256×256 | +|---|---|---| +| Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | +| DT64 bind (cold) | 2999 ns · 2 allocs | 9235 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1189 ns · 0 allocs · 0 B | 2447 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.1 µs | 177.4 µs | +| B preparation (VNNI panels) | 4844 ns · 8192 B written · 8320 B alloc | 88.1 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4009 ns · 65.4 GMAC/s · 0 allocs · 0 B tmp | 148.7 µs · 112.8 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 3567 ns · 73.5 GMAC/s | 107.0 µs · 156.7 GMAC/s | +| Staged AMX (dense A tiles) | 3939 ns · 1 allocs · 2048 B tmp | 115.9 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 34.8 µs · 23 allocs · 54272 B tmp | 950.7 µs · 263 allocs · 803840 B tmp | +| Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | +| CBLAS B materialized | 8192 B | 131072 B | +| FP32 accumulator (C) | 16384 B | 262144 B | +| End-to-end warm (resolve + direct, B reused) | 6229 ns | 127.7 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 20.9 µs | 271.0 µs | +| Warm repetitions (100, per run, one prepare) | 6228 ns | 155.7 µs | +| INT8 resolve (u8 lanes, one check) | 1129 ns | 1888 ns | +| INT8 stage A (region: one copy per lane) | 291 ns · 4096 B | 3457 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3263 ns | 24.0 µs | +| INT8 B for a region, in place from the table | 28 ns · 0 B written | 29 ns · 0 B written | +| INT8 materialize B (row-major i8) | 6097 ns · 4096 B | 99.2 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3471 ns · 75.5 GMAC/s · 3 allocs · 3072 B tmp | 77.8 µs · 215.7 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 1862 ns · 54% of the GEMM row | 29.9 µs · 38% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 10.7 µs | 85.3 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 4727 ns · 4096 B | 91.8 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 982 ns · 266.9 GMAC/s · 0 allocs · 0 B of A copied | 37.6 µs · 446.7 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 2890 ns | 42.6 µs | diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt new file mode 100644 index 000000000..4810f572d --- /dev/null +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt @@ -0,0 +1,35 @@ +Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K +Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). + +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 509.7 µs + +G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once 68.2 µs (3200 lookups) · prepare B once 36.9 µs · warm run 16.1 µs (0 lookups) · re-translate per operation 127.5 µs + +| Metric | 64×64 | 256×256 | +|---|---|---| +| Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | +| DT64 bind (cold) | 3179 ns · 2 allocs | 8522 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1246 ns · 0 allocs · 0 B | 2000 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.8 µs | 143.1 µs | +| B preparation (VNNI panels) | 5247 ns · 8192 B written · 8320 B alloc | 77.5 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4163 ns · 63.0 GMAC/s · 0 allocs · 0 B tmp | 134.9 µs · 124.4 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 3727 ns · 70.3 GMAC/s | 117.6 µs · 142.7 GMAC/s | +| Staged AMX (dense A tiles) | 4232 ns · 1 allocs · 2048 B tmp | 127.4 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 35.7 µs · 23 allocs · 54272 B tmp | 1031.9 µs · 263 allocs · 803840 B tmp | +| Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | +| CBLAS B materialized | 8192 B | 131072 B | +| FP32 accumulator (C) | 16384 B | 262144 B | +| End-to-end warm (resolve + direct, B reused) | 6387 ns | 144.6 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 20.9 µs | 265.5 µs | +| Warm repetitions (100, per run, one prepare) | 7247 ns | 155.3 µs | +| INT8 resolve (u8 lanes, one check) | 1222 ns | 1968 ns | +| INT8 stage A (region: one copy per lane) | 309 ns · 4096 B | 3504 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3460 ns | 24.6 µs | +| INT8 B for a region, in place from the table | 30 ns · 0 B written | 30 ns · 0 B written | +| INT8 materialize B (row-major i8) | 6477 ns · 4096 B | 102.7 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3590 ns · 73.0 GMAC/s · 3 allocs · 3072 B tmp | 80.1 µs · 209.4 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 1922 ns · 54% of the GEMM row | 29.4 µs · 37% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 5900 ns | 85.9 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 4979 ns · 4096 B | 106.3 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 1009 ns · 259.8 GMAC/s · 0 allocs · 0 B of A copied | 40.9 µs · 410.4 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 2996 ns | 36.1 µs | diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt new file mode 100644 index 000000000..dd0324c0d --- /dev/null +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt @@ -0,0 +1,35 @@ +Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K +Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). + +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 365.2 µs + +G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once 68.1 µs (3200 lookups) · prepare B once 38.0 µs · warm run 15.9 µs (0 lookups) · re-translate per operation 137.2 µs + +| Metric | 64×64 | 256×256 | +|---|---|---| +| Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | +| DT64 bind (cold) | 3183 ns · 2 allocs | 8797 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1225 ns · 0 allocs · 0 B | 2108 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.7 µs | 151.6 µs | +| B preparation (VNNI panels) | 5178 ns · 8192 B written · 8320 B alloc | 82.6 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4212 ns · 62.2 GMAC/s · 0 allocs · 0 B tmp | 139.5 µs · 120.3 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 3733 ns · 70.2 GMAC/s | 122.4 µs · 137.1 GMAC/s | +| Staged AMX (dense A tiles) | 5400 ns · 1 allocs · 2048 B tmp | 132.6 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 36.3 µs · 23 allocs · 54272 B tmp | 1052.7 µs · 263 allocs · 803840 B tmp | +| Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | +| CBLAS B materialized | 8192 B | 131072 B | +| FP32 accumulator (C) | 16384 B | 262144 B | +| End-to-end warm (resolve + direct, B reused) | 6386 ns | 147.5 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 16.4 µs | 265.2 µs | +| Warm repetitions (100, per run, one prepare) | 7146 ns | 150.0 µs | +| INT8 resolve (u8 lanes, one check) | 1218 ns | 1967 ns | +| INT8 stage A (region: one copy per lane) | 307 ns · 4096 B | 3589 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3854 ns | 24.6 µs | +| INT8 B for a region, in place from the table | 35 ns · 0 B written | 30 ns · 0 B written | +| INT8 materialize B (row-major i8) | 7387 ns · 4096 B | 102.3 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3971 ns · 66.0 GMAC/s · 3 allocs · 3072 B tmp | 78.8 µs · 212.8 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 1911 ns · 48% of the GEMM row | 29.5 µs · 37% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 5984 ns | 86.9 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 5003 ns · 4096 B | 79.1 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 1020 ns · 257.0 GMAC/s · 0 allocs · 0 B of A copied | 33.3 µs · 503.5 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 2994 ns | 40.6 µs | diff --git a/crates/dr-isa-stream-lab/src/bin/shape.rs b/crates/dr-isa-stream-lab/src/bin/shape.rs index 1954baf3c..0efb5767f 100644 --- a/crates/dr-isa-stream-lab/src/bin/shape.rs +++ b/crates/dr-isa-stream-lab/src/bin/shape.rs @@ -376,18 +376,19 @@ fn main() { assert_eq!(c8, want8); // The B packing gemm_i8 does inside every call, timed on its own: // per 16-column band, gather the band (K x 16) and VNNI-pack it with - // ndarray's own `vnni_pack_i8` (int8_gemm_amx_tiled_rb's steps, minus - // its three buffer allocations, which the GEMM row reports). + // `ndarray::simd::PackedI8B::pack_from` (the same layout and loop + // int8_gemm_amx_tiled_rb packs with, minus its three buffer + // allocations, which the GEMM row reports). let k8 = r8.k_pad; assert_eq!(b8.len(), k8 * dim, "B materialized row-major"); - let (mut band, mut packed) = (vec![0i8; k8 * 16], vec![0i8; k8 * 16]); + let (mut band, mut packed) = (vec![0i8; k8 * 16], ndarray::simd::PackedI8B::zeroed(k8)); let t8_pack = median_ns(|| { for j in (0..dim).step_by(16) { for kk in 0..k8 { band[kk * 16..kk * 16 + 16] .copy_from_slice(&b8[kk * dim + j..kk * dim + j + 16]); } - ndarray::hpc::amx_matmul::vnni_pack_i8(&band, &mut packed, k8, 16); + packed.pack_from(&band); black_box(&packed); } }); @@ -545,7 +546,8 @@ fn main() { let globals: Vec = (0..200).map(|k| global_of((k * 977 + 5) % 4096)).collect(); let bt = BTable::new(&b, 4096, dim); let tr = translate(&source, &globals).unwrap(); - let obj = VirtualObject::bind(&world, &socket, &fields, dim, tr.selection().clone()).unwrap(); + let obj = + VirtualObject::bind(&world, &socket, &fields, dim, tr.selection().clone()).unwrap(); let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); let prep = int8::PreparedI8B::prepare(&obj, &tb).unwrap(); let (mut a, mut c) = (Vec::new(), vec![0i32; obj.c_len()]); @@ -566,7 +568,8 @@ fn main() { assert_eq!(c, want); let t_per_op = median_ns(|| { let tr = translate(&source, &globals).unwrap(); - let o = VirtualObject::bind(&world, &socket, &fields, dim, tr.selection().clone()).unwrap(); + let o = + VirtualObject::bind(&world, &socket, &fields, dim, tr.selection().clone()).unwrap(); let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); let p = int8::PreparedI8B::prepare(&o, &tb).unwrap(); let l = int8::resolve(&o, &world, &socket).unwrap(); diff --git a/crates/dr-isa-stream-lab/src/g2l.rs b/crates/dr-isa-stream-lab/src/g2l.rs index 88617a17a..98558732d 100644 --- a/crates/dr-isa-stream-lab/src/g2l.rs +++ b/crates/dr-isa-stream-lab/src/g2l.rs @@ -124,7 +124,10 @@ impl OrderedSource { let lo = t as usize * TILE_POSITIONS; let hi = (lo + TILE_POSITIONS).min(self.order.len()); let globals: Vec = self.order[lo..hi].iter().map(|&i| global_of(i)).collect(); - (t, Some(LocalMap::prepare(Width::W256, &globals).expect("a tile holds 256 ids"))) + ( + t, + Some(LocalMap::prepare(Width::W256, &globals).expect("a tile holds 256 ids")), + ) }) .collect(); self.map.advance(1, 1, changes); @@ -267,6 +270,9 @@ pub struct TranslatedB<'a, T> { source: &'a OrderedSource, translation: &'a Translation, reads: Cell, + /// The last position resolved and its identity. Packing reads every + /// column of one row before the next, so one map read per row suffices. + last: Cell>, } impl<'a, T: BValues> TranslatedB<'a, T> { @@ -288,10 +294,11 @@ impl<'a, T: BValues> TranslatedB<'a, T> { source, translation, reads: Cell::new(0), + last: Cell::new(None), }) } - /// Values read through the translation so far. + /// Map reads made through the translation so far (one per row packed). pub fn reads(&self) -> u64 { self.reads.get() } @@ -299,6 +306,11 @@ impl<'a, T: BValues> TranslatedB<'a, T> { /// Source position → canonical identity, through the map's pointer table /// at the translation's frame (uncounted: the inverse read). fn identity_at(&self, position: usize) -> usize { + if let Some((p, identity)) = self.last.get() { + if p == position { + return identity; + } + } self.reads.set(self.reads.get() + 1); let tile = self.translation.tiles[position / TILE_POSITIONS]; let local = u8::try_from(position % TILE_POSITIONS).expect("a tile holds 256 positions"); @@ -307,7 +319,9 @@ impl<'a, T: BValues> TranslatedB<'a, T> { .map .map(&self.translation.frame, tile) .expect("the translation's own frame reads its own table"); - identity_of(map.global(local).expect("a translated position is mapped")) + let identity = identity_of(map.global(local).expect("a translated position is mapped")); + self.last.set(Some((position, identity))); + identity } } diff --git a/crates/dr-isa-stream-lab/tests/integration/g2l.rs b/crates/dr-isa-stream-lab/tests/integration/g2l.rs index ac75fad8d..2eee75c1f 100644 --- a/crates/dr-isa-stream-lab/tests/integration/g2l.rs +++ b/crates/dr-isa-stream-lab/tests/integration/g2l.rs @@ -4,9 +4,13 @@ use dr_isa_stream_lab::bind::{World, SPACE_A}; use dr_isa_stream_lab::fixture::Fixture; -use dr_isa_stream_lab::g2l::{global_of, identity_of, translate, G2lRefusal, OrderedSource, TranslatedB}; +use dr_isa_stream_lab::g2l::{ + global_of, identity_of, translate, G2lRefusal, OrderedSource, TranslatedB, +}; use dr_isa_stream_lab::int8::{self, a_u8, b_i8}; -use dr_isa_stream_lab::shape::{BOperand, BTable, ReuseRefusal, Selection, ShapeRefusal, VirtualObject}; +use dr_isa_stream_lab::shape::{ + BOperand, BTable, ReuseRefusal, Selection, ShapeRefusal, VirtualObject, +}; use lance_graph_dt64::dr::{FrameRefusal, LocateRefusal}; use lance_graph_dt64::{Extent, LaneRefusal}; @@ -76,7 +80,8 @@ fn a_translation_is_exact_and_reused_with_zero_further_lookups() { let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); let prep = int8::PreparedI8B::prepare(&obj, &tb).unwrap(); let (lookups, reads) = (source.lookups(), tb.reads()); - assert!(reads > 0); + // Packing resolves each selected row once, not once per column. + assert_eq!(reads, globals.len() as u64); let (mut a, mut c) = (Vec::new(), vec![0i32; obj.c_len()]); for run in 0..5 { @@ -113,7 +118,8 @@ fn a_reorder_refuses_everything_prepared_under_the_old_order() { let (tr, obj, prep) = { let socket = source.socket(DIM); let tr = translate(&source, &globals).unwrap(); - let obj = VirtualObject::bind(&world, &socket, &fields(), DIM, tr.selection().clone()).unwrap(); + let obj = + VirtualObject::bind(&world, &socket, &fields(), DIM, tr.selection().clone()).unwrap(); let tb = TranslatedB::new(&bt, &source, &tr).unwrap(); let prep = int8::PreparedI8B::prepare(&obj, &tb).unwrap(); (tr, obj, prep) @@ -123,7 +129,9 @@ fn a_reorder_refuses_everything_prepared_under_the_old_order() { let socket = source.socket(DIM); assert!(matches!( int8::resolve(&obj, &world, &socket), - Err(ShapeRefusal::Lane(LaneRefusal::Frame(FrameRefusal::CoordinatesChanged))) + Err(ShapeRefusal::Lane(LaneRefusal::Frame( + FrameRefusal::CoordinatesChanged + ))) )); assert!(matches!(tr.check(&source), Err(G2lRefusal::Stale { .. }))); assert!(matches!( @@ -134,7 +142,8 @@ fn a_reorder_refuses_everything_prepared_under_the_old_order() { // A fresh translation under the new order is exact again. let tr2 = translate(&source, &globals).unwrap(); - let obj2 = VirtualObject::bind(&world, &socket, &fields(), DIM, tr2.selection().clone()).unwrap(); + let obj2 = + VirtualObject::bind(&world, &socket, &fields(), DIM, tr2.selection().clone()).unwrap(); let tb2 = TranslatedB::new(&bt, &source, &tr2).unwrap(); let prep2 = int8::PreparedI8B::prepare(&obj2, &tb2).unwrap(); let lanes = int8::resolve(&obj2, &world, &socket).unwrap(); @@ -184,7 +193,8 @@ fn a_prepared_b_belongs_to_its_translation() { let bt = BTable::new(&B, N, DIM); let tr1 = translate(&source, &wanted()).unwrap(); let tr2 = translate(&source, &wanted()).unwrap(); - let obj = VirtualObject::bind(&world, &socket, &fields(), DIM, tr1.selection().clone()).unwrap(); + let obj = + VirtualObject::bind(&world, &socket, &fields(), DIM, tr1.selection().clone()).unwrap(); let tb1 = TranslatedB::new(&bt, &source, &tr1).unwrap(); let tb2 = TranslatedB::new(&bt, &source, &tr2).unwrap(); let prep = int8::PreparedI8B::prepare(&obj, &tb1).unwrap(); From cf7abf06602322dcf3ba9b1423bfe29458828bfb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:42:55 +0000 Subject: [PATCH 6/9] dr-isa-stream-lab: refuse a translated identity past the inner B table TranslatedB::extent() reported the source's position range, so an identity past the inner table passed admit_operand and packing indexed past the table. extent() now reports the inner operand's own extent, and BValues gains highest_row(selection), which TranslatedB overrides to resolve each selected position to the identity it reads. admit_operand then refuses with ShapeRefusal::OperandShape before any packing (#1990's contract). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/dr-isa-stream-lab/src/g2l.rs | 50 ++++++++++++++----- crates/dr-isa-stream-lab/src/shape.rs | 10 +++- .../tests/integration/g2l.rs | 34 +++++++++++++ 3 files changed, 81 insertions(+), 13 deletions(-) diff --git a/crates/dr-isa-stream-lab/src/g2l.rs b/crates/dr-isa-stream-lab/src/g2l.rs index 98558732d..c94104cea 100644 --- a/crates/dr-isa-stream-lab/src/g2l.rs +++ b/crates/dr-isa-stream-lab/src/g2l.rs @@ -270,6 +270,7 @@ pub struct TranslatedB<'a, T> { source: &'a OrderedSource, translation: &'a Translation, reads: Cell, + admission_reads: Cell, /// The last position resolved and its identity. Packing reads every /// column of one row before the next, so one map read per row suffices. last: Cell>, @@ -294,6 +295,7 @@ impl<'a, T: BValues> TranslatedB<'a, T> { source, translation, reads: Cell::new(0), + admission_reads: Cell::new(0), last: Cell::new(None), }) } @@ -303,8 +305,25 @@ impl<'a, T: BValues> TranslatedB<'a, T> { self.reads.get() } + /// Map reads made while admitting the operand (one per selected row, + /// to find the highest inner row it reads), counted apart from + /// [`Self::reads`]. + pub fn admission_reads(&self) -> u64 { + self.admission_reads.get() + } + /// Source position → canonical identity, through the map's pointer table - /// at the translation's frame (uncounted: the inverse read). + /// at the translation's frame (the inverse read). A position outside the + /// translation's tiles, or unmapped, is `None`. + fn resolve(&self, position: usize) -> Option { + let tile = *self.translation.tiles.get(position / TILE_POSITIONS)?; + let local = u8::try_from(position % TILE_POSITIONS).ok()?; + let map = self.source.map.map(&self.translation.frame, tile).ok()?; + Some(identity_of(map.global(local)?)) + } + + /// [`Self::resolve`] for packing, cached per row. Packing runs only after + /// [`crate::shape::admit_operand`] resolved every selected position. fn identity_at(&self, position: usize) -> usize { if let Some((p, identity)) = self.last.get() { if p == position { @@ -312,14 +331,9 @@ impl<'a, T: BValues> TranslatedB<'a, T> { } } self.reads.set(self.reads.get() + 1); - let tile = self.translation.tiles[position / TILE_POSITIONS]; - let local = u8::try_from(position % TILE_POSITIONS).expect("a tile holds 256 positions"); - let map = self - .source - .map - .map(&self.translation.frame, tile) - .expect("the translation's own frame reads its own table"); - let identity = identity_of(map.global(local).expect("a translated position is mapped")); + let identity = self + .resolve(position) + .expect("admission resolved every selected position"); self.last.set(Some((position, identity))); identity } @@ -341,9 +355,21 @@ impl BValues for TranslatedB<'_, T> { fn i8_at(&self, i: usize, j: usize) -> i8 { self.inner.i8_at(self.identity_at(i), j) } + /// The inner operand's own extent: its rows are identities, not source + /// positions, so a translated identity past the inner table is refused + /// through [`Self::highest_row`] (#1990's OperandShape contract). fn extent(&self) -> Option<(usize, usize)> { - self.inner - .extent() - .map(|(_, cols)| (self.source.len(), cols)) + self.inner.extent() + } + /// The highest identity any selected position reads. An unmapped + /// position reads as `usize::MAX`, so admission refuses it too. + fn highest_row(&self, selection: &Selection) -> Option { + let mut highest = None; + selection.for_each(|_, position| { + self.admission_reads.set(self.admission_reads.get() + 1); + let identity = self.resolve(position).unwrap_or(usize::MAX); + highest = highest.max(Some(identity)); + }); + highest } } diff --git a/crates/dr-isa-stream-lab/src/shape.rs b/crates/dr-isa-stream-lab/src/shape.rs index 6e0ac63c8..d0696e3ae 100644 --- a/crates/dr-isa-stream-lab/src/shape.rs +++ b/crates/dr-isa-stream-lab/src/shape.rs @@ -437,6 +437,14 @@ pub trait BValues { fn extent(&self) -> Option<(usize, usize)> { None } + /// The highest row of the table behind [`Self::extent`] that reading + /// `B[i][..]` for every selected `i` touches. For an operand read + /// directly that is the selection's last identity; an operand read + /// through a translation (`crate::g2l::TranslatedB`) maps each selected + /// position to the identity it reads. + fn highest_row(&self, selection: &Selection) -> Option { + selection.last() + } } /// Refuses an operand that cannot supply every `B[i][j]` the object reads: @@ -447,7 +455,7 @@ pub fn admit_operand(obj: &VirtualObject, b: &impl BValues) -> Result<(), ShapeR let Some((rows, table_cols)) = b.extent() else { return Ok(()); }; - let last = obj.selection.last().unwrap_or(0); + let last = b.highest_row(&obj.selection).unwrap_or(0); if obj.cols > table_cols || last >= rows { return Err(ShapeRefusal::OperandShape { cols: obj.cols, diff --git a/crates/dr-isa-stream-lab/tests/integration/g2l.rs b/crates/dr-isa-stream-lab/tests/integration/g2l.rs index 2eee75c1f..88ae35fb5 100644 --- a/crates/dr-isa-stream-lab/tests/integration/g2l.rs +++ b/crates/dr-isa-stream-lab/tests/integration/g2l.rs @@ -207,3 +207,37 @@ fn a_prepared_b_belongs_to_its_translation() { int8::staged_prepared(&obj, &lanes, &prep, &tb1, &mut Vec::new(), &mut c).unwrap(); assert_eq!(c, oracle(&fx, &wanted())); } + +/// #1990's OperandShape contract survives translation. The inner B table +/// covers identities `0..100` only; the source holds all `N`. One selected +/// global maps to identity 3000, past the table. The source position it +/// lands on is inside `0..N`, so checking positions against the source's +/// length would admit it and packing would read past the table. Preparation +/// must refuse it, with the inner row and the table's extent, before any +/// packing. +#[test] +fn a_translated_identity_past_the_inner_table_is_refused_before_packing() { + let fx = fx(); + let source = OrderedSource::new(&fx, DIM, order(1237, 11)); + let socket = source.socket(DIM); + let short = BTable::new(&B, 100, DIM); + let globals = vec![global_of(3), global_of(3000), global_of(42)]; + let tr = translate(&source, &globals).unwrap(); + let w = world(); + let obj = VirtualObject::bind(&w, &socket, &fields(), DIM, tr.selection().clone()).unwrap(); + let tb = TranslatedB::new(&short, &source, &tr).unwrap(); + // Anti-vacuity: the positions really are inside the source. + assert!(tr.selection().last().unwrap() < N); + match int8::PreparedI8B::prepare(&obj, &tb) { + Err(ShapeRefusal::OperandShape { last, rows, .. }) => assert_eq!((last, rows), (3000, 100)), + Err(other) => panic!("expected OperandShape, got {other:?}"), + Ok(_) => panic!("expected OperandShape, got a prepared B"), + } + assert_eq!(tb.reads(), 0, "refused before packing read anything"); + // Silence twin: identities that all fit the table are admitted. + let ok = vec![global_of(3), global_of(99), global_of(42)]; + let tr = translate(&source, &ok).unwrap(); + let obj = VirtualObject::bind(&w, &socket, &fields(), DIM, tr.selection().clone()).unwrap(); + let tb = TranslatedB::new(&short, &source, &tr).unwrap(); + int8::PreparedI8B::prepare(&obj, &tb).expect("every identity is inside the table"); +} From 42ead5d98426e184e3e169f674ad44595ec829df Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:44:56 +0000 Subject: [PATCH 7/9] dr-isa-stream-lab: report G2L costs one by one The G2L bench now reports separately: G2L lookups and translation time; admission and packing map reads during B preparation; B bytes packed and preparation time; A staging bytes and time per warm run; GEMM-only time; and the complete warm run. GEMM alone is 0.71-0.86 us; staging the sparse selection into a padded A is 11-14 us of a 20-24 us warm run, so zero lookups is not zero data movement. BENCH-SHAPE, the board entry and the contract doc record that K order changes under translation and is validated only for exact integer GEMM. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .../board/entries/2026-11-10-dt64-g2l-lab.md | 18 ++++-- .../dr-dt64-locality-execution-contract.md | 2 +- crates/dr-isa-stream-lab/BENCH-SHAPE.md | 51 +++++++++++++---- .../results/shape-2026-11-10-g2l-run1.txt | 56 +++++++++++-------- .../results/shape-2026-11-10-g2l-run2.txt | 54 ++++++++++-------- .../results/shape-2026-11-10-g2l-run3.txt | 54 ++++++++++-------- crates/dr-isa-stream-lab/src/bin/shape.rs | 43 ++++++++++++-- crates/dr-isa-stream-lab/src/int8.rs | 5 ++ 8 files changed, 189 insertions(+), 94 deletions(-) diff --git a/.claude/board/entries/2026-11-10-dt64-g2l-lab.md b/.claude/board/entries/2026-11-10-dt64-g2l-lab.md index 9d68f3f35..652e1569e 100644 --- a/.claude/board/entries/2026-11-10-dt64-g2l-lab.md +++ b/.claude/board/entries/2026-11-10-dt64-g2l-lab.md @@ -11,13 +11,21 @@ through the translation; every run then reads the source's lanes in place with zero lookups. Cross-space reads without a translation stay refused, and the translation does not grant Gather (ADMISSION B #1640). -**Result** (one AMX VM, three runs): translate once 68–85 µs (3200 lookups), -prepare B once 37–47 µs, warm run 16–22 µs with 0 lookups; re-translating per -operation 128–166 µs (about 8×). +**Result** (one AMX VM, three runs, each quantity on its own): translate +once 83–85 µs (3200 lookups); prepare B once 46–48 µs (200 admission + 200 +packing map reads, 16,384 B bytes); stage A 11–14 µs per warm run (12,800 +useful bytes into a 16,384-byte padded buffer); GEMM only 0.71–0.86 µs; warm +run 20–24 µs with 0 lookups; re-translating per operation 157–186 µs. Zero +lookups is not zero movement: the sparse selection is still gathered into A on +every run, at about 15× the GEMM. -**Falsifiers:** oracle in global ids only; reorder refuses binding, +**Scope:** K is reordered by the translation. That is validated only for exact +integer GEMM (order-independent sum absent overflow), not for order-sensitive +operators. + +**Falsifiers:** oracle in global ids only; an identity past the inner B table refused with OperandShape before packing (#1990 contract); reorder refuses binding, translation and operand; stale mapping refused; prepared B bound to its -translation; unknown and repeated ids refused. Four disable runs, each red. +translation; unknown and repeated ids refused. Five disable runs, each red. **Governing question:** the location of each selected id is resolved once per translation; B's per-row identity once per preparation; per operation, diff --git a/.claude/knowledge/dr-dt64-locality-execution-contract.md b/.claude/knowledge/dr-dt64-locality-execution-contract.md index d1ca70adf..9bf3a5ad9 100644 --- a/.claude/knowledge/dr-dt64-locality-execution-contract.md +++ b/.claude/knowledge/dr-dt64-locality-execution-contract.md @@ -122,7 +122,7 @@ field mapping, operand identity and version. | Lab resolve on `borrow_all` (follow-up) | MEASURED, not merged | Resolve at 256 fields 1.6–2.0 µs vs 13.8–13.9 µs (#1987 as first measured) and 26–28 µs (`main` after the provenance fix); the old cost was `LaneBinding::addr`, a linear scan called per field | | INT8 vs BF16 over the same object (follow-up) | MEASURED, not merged | `u8 × i8 → i32` end to end 66–101 µs vs BF16 103–160 µs at 256 fields on one VM; INT8 still copies A (5–7 µs) and re-packs B per call (23–29 µs, 36–38% of its GEMM), because ndarray has no strided-A / prepacked-B INT8 entry | | #1987's "B preparation ≈ GEMM" | REJECTED-BY-MEASUREMENT | That timer included the synthetic operand's hash; from a table built once, prepared B costs about a third of the GEMM | -| G2L across different source orders (D-DT64-G2L-1) | MEASURED, lab | One translation under one `MapTable` frame (counted lookups), reused across runs with 0 lookups; warm run 16–22 µs vs 128–166 µs re-translating per operation; reorder, stale mapping and foreign translation refused. Still OPEN for production: who owns the translation, and frames across a restore (#1993) | +| G2L across different source orders (D-DT64-G2L-1) | MEASURED, lab | One translation under one `MapTable` frame (counted lookups), reused across runs with 0 lookups; warm run 20–24 µs (of which A staging 11–14 µs, GEMM 0.7–0.9 µs) vs 157–186 µs re-translating per operation; reorder, stale mapping, foreign translation and an identity past the inner operand refused. Exact integer GEMM only: K order changes under translation. Still OPEN for production: who owns the translation, and frames across a restore (#1993) | | Foundry operational-ontology parity | HYPOTHESIS | Research goal; nothing here establishes it | All timings: one VM (Xeon with AMX-BF16, 4 vCPU), single-threaded, medians of diff --git a/crates/dr-isa-stream-lab/BENCH-SHAPE.md b/crates/dr-isa-stream-lab/BENCH-SHAPE.md index 1d5483ce0..3307f1aa5 100644 --- a/crates/dr-isa-stream-lab/BENCH-SHAPE.md +++ b/crates/dr-isa-stream-lab/BENCH-SHAPE.md @@ -367,20 +367,46 @@ on co-addressed sets. So: the prepared B, with **zero** lookups. A sum over K does not depend on K's order, so the result is the consumer's answer. -One AMX VM, three runs (`results/shape-2026-11-10-g2l-run*.txt`), 64 fields × -64 columns, 200 global ids: +One AMX VM (`int8_tile_gemm_tier()` = AMX TDPBUSD), three runs +(`results/shape-2026-11-10-g2l-run*.txt`), 64 fields × 64 columns, 200 global +ids. Each quantity is reported on its own; ranges are over the three runs: -| Step | Time | Lookups | +| Step | Time | Movement | |---|---|---| -| translate once | 68–85 µs | 3200 (200 ids × 16 tiles) | -| prepare B once | 37–47 µs | 200 map reads | -| **warm run** (resolve + stage A + GEMM) | **16–22 µs** | **0** | -| re-translate on every operation (the anti-pattern) | 128–166 µs | 3200 + 200 | - -Re-translating per operation costs about 8× the warm run. The first version of -`TranslatedB` resolved position → identity once per (row, column) and took -241–255 µs to prepare B: 64× the map reads needed. One read per row fixed it -(test-pinned: exactly 200 reads). +| translate once | 83–85 µs | 3200 G2L lookups (200 ids × 16 tiles) | +| prepare B once (admit + pack) | 46–48 µs | 200 admission map reads + 200 packing map reads; 16,384 B bytes packed | +| stage A, per warm run | 11–14 µs | 12,800 useful bytes copied into a 16,384-byte padded buffer | +| GEMM only | 0.71–0.86 µs | — | +| **warm run** (resolve + stage A + GEMM) | **20–24 µs** | 0 G2L lookups, 0 map reads, 12,800 A bytes staged | +| re-translate per operation | 157–186 µs | 3200 lookups + 400 map reads + B pack + A stage | + +**Zero lookups is not zero data movement.** The selection is sparse in the +source's order, so a warm run still gathers 200 values per field into a dense, +K-padded A: 12,800 useful bytes into a 16,384-byte buffer, every run. That +copy costs about 15× the GEMM it feeds. What the translation removed is the +global → local resolution; what it did not remove is the gather of a scattered +selection. Reading A in place needs the selection to be a region in the +source's order (as in the INT8 direct arm), which this fixture is not. + +**Order of K.** Translation reorders K: B is packed in the source's row order, +not the consumer's. That is validated here only for exact integer GEMM, where +`Σ_k a·b` does not depend on the order of summation as long as nothing +overflows (i32 accumulation of 200 products of |a| ≤ 255, |b| ≤ 8, far inside +range). It does not carry over to order-sensitive operators: floating-point +reduction, saturating arithmetic, prefix scans, argmax with tie-breaking, or +anything that reads K positionally. + +**Operand extent.** `TranslatedB` reports the inner operand's own extent (its +rows are identities) and maps every selected source position to the identity +it reads (`BValues::highest_row`). An identity past the inner table is +refused with `ShapeRefusal::OperandShape` before packing reads anything. +Before this fix it reported the source's position range and packing indexed +past the table (`tests/integration/g2l.rs`, +`a_translated_identity_past_the_inner_table_is_refused_before_packing`). + +The first version of `TranslatedB` resolved position → identity once per +(row, column) and took 241–255 µs to prepare B: 64× the map reads needed. One +read per row fixed it (test-pinned: exactly 200 packing reads). Falsifiers (`tests/integration/g2l.rs`), each disable-verified red: @@ -391,6 +417,7 @@ Falsifiers (`tests/integration/g2l.rs`), each disable-verified red: | a stale mapping (lanes reordered, map not) is refused | map-vs-source coordinate check removed | | a prepared B from one translation is refused under another | translated identity ignoring the translation | | the untranslated cross-space read stays refused | (DT64's existing check) | +| an identity past the inner B table is refused before packing (OperandShape) | `TranslatedB::highest_row` removed: packing reads past the table | | unknown and repeated ids are refused | — | A measurement trap worth keeping: after the disable runs, the benchmark binary diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt index b6e92b876..a43091a22 100644 --- a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run1.txt @@ -1,35 +1,43 @@ Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). -Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 380.1 µs +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 390.4 µs -G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once 84.6 µs (3200 lookups) · prepare B once 47.3 µs · warm run 21.5 µs (0 lookups) · re-translate per operation 165.6 µs +G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles, AMX TDPBUSD): +| Step | Time | Movement | +|---|---|---| +| translate once | 84.4 µs | 3200 G2L lookups | +| prepare B once (admit + pack) | 48.1 µs | 200 admission map reads, 200 packing map reads, 16384 B bytes packed | +| stage A (per warm run) | 13.6 µs | 12800 useful bytes copied into a 16384-byte padded buffer | +| GEMM only | 855 ns | — | +| warm run (resolve + stage A + GEMM) | 23.5 µs | 0 G2L lookups, 0 map reads, 12800 A bytes staged | +| re-translate per operation | 185.9 µs | 3200 lookups + 400 map reads + B pack + A stage | | Metric | 64×64 | 256×256 | |---|---|---| | Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | -| DT64 bind (cold) | 2999 ns · 2 allocs | 9235 ns · 6 allocs | -| Resolve: one check + borrows (warm) | 1189 ns · 0 allocs · 0 B | 2447 ns · 0 allocs · 0 B | -| Resolve: `lane()` per field (warm) | 11.1 µs | 177.4 µs | -| B preparation (VNNI panels) | 4844 ns · 8192 B written · 8320 B alloc | 88.1 µs · 131072 B written · 131584 B alloc | -| Direct AMX (A in place) | 4009 ns · 65.4 GMAC/s · 0 allocs · 0 B tmp | 148.7 µs · 112.8 GMAC/s · 0 allocs · 0 B tmp | -| Direct AMX, pitch 65,568 (not 2^k) | 3567 ns · 73.5 GMAC/s | 107.0 µs · 156.7 GMAC/s | -| Staged AMX (dense A tiles) | 3939 ns · 1 allocs · 2048 B tmp | 115.9 µs · 1 allocs · 8192 B tmp | -| ndarray CBLAS BF16 | 34.8 µs · 23 allocs · 54272 B tmp | 950.7 µs · 263 allocs · 803840 B tmp | +| DT64 bind (cold) | 3086 ns · 2 allocs | 8644 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1171 ns · 0 allocs · 0 B | 2025 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.4 µs | 154.0 µs | +| B preparation (VNNI panels) | 4796 ns · 8192 B written · 8320 B alloc | 79.5 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4110 ns · 63.8 GMAC/s · 0 allocs · 0 B tmp | 142.1 µs · 118.1 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 3697 ns · 70.9 GMAC/s | 146.5 µs · 114.6 GMAC/s | +| Staged AMX (dense A tiles) | 4153 ns · 1 allocs · 2048 B tmp | 153.3 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 36.2 µs · 23 allocs · 54272 B tmp | 1036.1 µs · 263 allocs · 803840 B tmp | | Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | | CBLAS B materialized | 8192 B | 131072 B | | FP32 accumulator (C) | 16384 B | 262144 B | -| End-to-end warm (resolve + direct, B reused) | 6229 ns | 127.7 µs | -| End-to-end cold (bind + prepare + resolve + direct) | 20.9 µs | 271.0 µs | -| Warm repetitions (100, per run, one prepare) | 6228 ns | 155.7 µs | -| INT8 resolve (u8 lanes, one check) | 1129 ns | 1888 ns | -| INT8 stage A (region: one copy per lane) | 291 ns · 4096 B | 3457 ns · 65536 B | -| INT8 stage A, per-value walk (baseline) | 3263 ns | 24.0 µs | -| INT8 B for a region, in place from the table | 28 ns · 0 B written | 29 ns · 0 B written | -| INT8 materialize B (row-major i8) | 6097 ns · 4096 B | 99.2 µs · 65536 B | -| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3471 ns · 75.5 GMAC/s · 3 allocs · 3072 B tmp | 77.8 µs · 215.7 GMAC/s · 3 allocs · 12288 B tmp | -| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 1862 ns · 54% of the GEMM row | 29.9 µs · 38% of the GEMM row | -| INT8 end-to-end warm (resolve + stage + GEMM) | 10.7 µs | 85.3 µs | -| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 4727 ns · 4096 B | 91.8 µs · 65536 B | -| INT8 direct GEMM (A in place, prepared B) | 982 ns · 266.9 GMAC/s · 0 allocs · 0 B of A copied | 37.6 µs · 446.7 GMAC/s · 0 allocs · 0 B of A copied | -| INT8 direct end-to-end warm (resolve + GEMM) | 2890 ns | 42.6 µs | +| End-to-end warm (resolve + direct, B reused) | 6479 ns | 144.2 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 16.0 µs | 264.5 µs | +| Warm repetitions (100, per run, one prepare) | 6394 ns | 150.0 µs | +| INT8 resolve (u8 lanes, one check) | 1208 ns | 1977 ns | +| INT8 stage A (region: one copy per lane) | 308 ns · 4096 B | 4069 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3452 ns | 27.3 µs | +| INT8 B for a region, in place from the table | 30 ns · 0 B written | 30 ns · 0 B written | +| INT8 materialize B (row-major i8) | 6545 ns · 4096 B | 104.9 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3556 ns · 73.7 GMAC/s · 3 allocs · 3072 B tmp | 80.8 µs · 207.7 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 2110 ns · 59% of the GEMM row | 29.5 µs · 36% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 5827 ns | 85.9 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 5370 ns · 4096 B | 80.2 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 1007 ns · 260.3 GMAC/s · 0 allocs · 0 B of A copied | 43.6 µs · 385.2 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 3019 ns | 44.0 µs | diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt index 4810f572d..f9218e09b 100644 --- a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run2.txt @@ -1,35 +1,43 @@ Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). -Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 509.7 µs +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 346.8 µs -G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once 68.2 µs (3200 lookups) · prepare B once 36.9 µs · warm run 16.1 µs (0 lookups) · re-translate per operation 127.5 µs +G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles, AMX TDPBUSD): +| Step | Time | Movement | +|---|---|---| +| translate once | 84.9 µs | 3200 G2L lookups | +| prepare B once (admit + pack) | 47.7 µs | 200 admission map reads, 200 packing map reads, 16384 B bytes packed | +| stage A (per warm run) | 13.7 µs | 12800 useful bytes copied into a 16384-byte padded buffer | +| GEMM only | 857 ns | — | +| warm run (resolve + stage A + GEMM) | 22.6 µs | 0 G2L lookups, 0 map reads, 12800 A bytes staged | +| re-translate per operation | 183.3 µs | 3200 lookups + 400 map reads + B pack + A stage | | Metric | 64×64 | 256×256 | |---|---|---| | Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | -| DT64 bind (cold) | 3179 ns · 2 allocs | 8522 ns · 6 allocs | -| Resolve: one check + borrows (warm) | 1246 ns · 0 allocs · 0 B | 2000 ns · 0 allocs · 0 B | -| Resolve: `lane()` per field (warm) | 11.8 µs | 143.1 µs | -| B preparation (VNNI panels) | 5247 ns · 8192 B written · 8320 B alloc | 77.5 µs · 131072 B written · 131584 B alloc | -| Direct AMX (A in place) | 4163 ns · 63.0 GMAC/s · 0 allocs · 0 B tmp | 134.9 µs · 124.4 GMAC/s · 0 allocs · 0 B tmp | -| Direct AMX, pitch 65,568 (not 2^k) | 3727 ns · 70.3 GMAC/s | 117.6 µs · 142.7 GMAC/s | -| Staged AMX (dense A tiles) | 4232 ns · 1 allocs · 2048 B tmp | 127.4 µs · 1 allocs · 8192 B tmp | -| ndarray CBLAS BF16 | 35.7 µs · 23 allocs · 54272 B tmp | 1031.9 µs · 263 allocs · 803840 B tmp | +| DT64 bind (cold) | 3183 ns · 2 allocs | 8677 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1227 ns · 0 allocs · 0 B | 2075 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 12.0 µs | 152.3 µs | +| B preparation (VNNI panels) | 4869 ns · 8192 B written · 8320 B alloc | 79.5 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4601 ns · 57.0 GMAC/s · 0 allocs · 0 B tmp | 148.0 µs · 113.4 GMAC/s · 0 allocs · 0 B tmp | +| Direct AMX, pitch 65,568 (not 2^k) | 4130 ns · 63.5 GMAC/s | 130.9 µs · 128.2 GMAC/s | +| Staged AMX (dense A tiles) | 4416 ns · 1 allocs · 2048 B tmp | 146.5 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 36.4 µs · 23 allocs · 54272 B tmp | 1025.9 µs · 263 allocs · 803840 B tmp | | Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | | CBLAS B materialized | 8192 B | 131072 B | | FP32 accumulator (C) | 16384 B | 262144 B | -| End-to-end warm (resolve + direct, B reused) | 6387 ns | 144.6 µs | -| End-to-end cold (bind + prepare + resolve + direct) | 20.9 µs | 265.5 µs | -| Warm repetitions (100, per run, one prepare) | 7247 ns | 155.3 µs | -| INT8 resolve (u8 lanes, one check) | 1222 ns | 1968 ns | -| INT8 stage A (region: one copy per lane) | 309 ns · 4096 B | 3504 ns · 65536 B | -| INT8 stage A, per-value walk (baseline) | 3460 ns | 24.6 µs | +| End-to-end warm (resolve + direct, B reused) | 6814 ns | 150.4 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 16.0 µs | 262.6 µs | +| Warm repetitions (100, per run, one prepare) | 6583 ns | 154.3 µs | +| INT8 resolve (u8 lanes, one check) | 1195 ns | 1975 ns | +| INT8 stage A (region: one copy per lane) | 309 ns · 4096 B | 4934 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3439 ns | 30.9 µs | | INT8 B for a region, in place from the table | 30 ns · 0 B written | 30 ns · 0 B written | -| INT8 materialize B (row-major i8) | 6477 ns · 4096 B | 102.7 µs · 65536 B | -| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3590 ns · 73.0 GMAC/s · 3 allocs · 3072 B tmp | 80.1 µs · 209.4 GMAC/s · 3 allocs · 12288 B tmp | -| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 1922 ns · 54% of the GEMM row | 29.4 µs · 37% of the GEMM row | -| INT8 end-to-end warm (resolve + stage + GEMM) | 5900 ns | 85.9 µs | -| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 4979 ns · 4096 B | 106.3 µs · 65536 B | -| INT8 direct GEMM (A in place, prepared B) | 1009 ns · 259.8 GMAC/s · 0 allocs · 0 B of A copied | 40.9 µs · 410.4 GMAC/s · 0 allocs · 0 B of A copied | -| INT8 direct end-to-end warm (resolve + GEMM) | 2996 ns | 36.1 µs | +| INT8 materialize B (row-major i8) | 6267 ns · 4096 B | 102.1 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3584 ns · 73.1 GMAC/s · 3 allocs · 3072 B tmp | 78.7 µs · 213.0 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 2105 ns · 59% of the GEMM row | 29.6 µs · 38% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 5906 ns | 89.7 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 5327 ns · 4096 B | 79.1 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 1011 ns · 259.3 GMAC/s · 0 allocs · 0 B of A copied | 50.3 µs · 333.2 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 3064 ns | 54.3 µs | diff --git a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt index dd0324c0d..a3c4a884f 100644 --- a/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt +++ b/crates/dr-isa-stream-lab/results/shape-2026-11-10-g2l-run3.txt @@ -1,35 +1,43 @@ Host: Intel(R) Xeon(R) Processor @ 2.10GHz; 4 logical CPUs; cache (cpu0): L1 Data 48K, L1 Instruction 32K, L2 Unified 2048K, L3 Unified 266240K Tile tier: AMX TDPBF16PS. Single-threaded. 41 warm runs per row (median). -Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 365.2 µs +Sparse demand (256 fields x 256 cols, 40 scattered identities): logical selection 40 · K padded 64 · lanes borrowed 256 · A bytes copied 20480 · dense dot-product steps 512 · staged AMX 364.3 µs -G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once 68.1 µs (3200 lookups) · prepare B once 38.0 µs · warm run 15.9 µs (0 lookups) · re-translate per operation 137.2 µs +G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles, AMX TDPBUSD): +| Step | Time | Movement | +|---|---|---| +| translate once | 82.7 µs | 3200 G2L lookups | +| prepare B once (admit + pack) | 46.4 µs | 200 admission map reads, 200 packing map reads, 16384 B bytes packed | +| stage A (per warm run) | 10.9 µs | 12800 useful bytes copied into a 16384-byte padded buffer | +| GEMM only | 714 ns | — | +| warm run (resolve + stage A + GEMM) | 20.4 µs | 0 G2L lookups, 0 map reads, 12800 A bytes staged | +| re-translate per operation | 157.2 µs | 3200 lookups + 400 map reads + B pack + A stage | | Metric | 64×64 | 256×256 | |---|---|---| | Output tiles / K=32 dot-product steps | 16 / 32 | 256 / 2048 | -| DT64 bind (cold) | 3183 ns · 2 allocs | 8797 ns · 6 allocs | -| Resolve: one check + borrows (warm) | 1225 ns · 0 allocs · 0 B | 2108 ns · 0 allocs · 0 B | -| Resolve: `lane()` per field (warm) | 11.7 µs | 151.6 µs | -| B preparation (VNNI panels) | 5178 ns · 8192 B written · 8320 B alloc | 82.6 µs · 131072 B written · 131584 B alloc | -| Direct AMX (A in place) | 4212 ns · 62.2 GMAC/s · 0 allocs · 0 B tmp | 139.5 µs · 120.3 GMAC/s · 0 allocs · 0 B tmp | +| DT64 bind (cold) | 3184 ns · 2 allocs | 8665 ns · 6 allocs | +| Resolve: one check + borrows (warm) | 1223 ns · 0 allocs · 0 B | 2036 ns · 0 allocs · 0 B | +| Resolve: `lane()` per field (warm) | 11.8 µs | 152.1 µs | +| B preparation (VNNI panels) | 4898 ns · 8192 B written · 8320 B alloc | 79.4 µs · 131072 B written · 131584 B alloc | +| Direct AMX (A in place) | 4271 ns · 61.4 GMAC/s · 0 allocs · 0 B tmp | 141.6 µs · 118.5 GMAC/s · 0 allocs · 0 B tmp | | Direct AMX, pitch 65,568 (not 2^k) | 3733 ns · 70.2 GMAC/s | 122.4 µs · 137.1 GMAC/s | -| Staged AMX (dense A tiles) | 5400 ns · 1 allocs · 2048 B tmp | 132.6 µs · 1 allocs · 8192 B tmp | -| ndarray CBLAS BF16 | 36.3 µs · 23 allocs · 54272 B tmp | 1052.7 µs · 263 allocs · 803840 B tmp | +| Staged AMX (dense A tiles) | 4085 ns · 1 allocs · 2048 B tmp | 134.9 µs · 1 allocs · 8192 B tmp | +| ndarray CBLAS BF16 | 37.3 µs · 23 allocs · 54272 B tmp | 1016.1 µs · 263 allocs · 803840 B tmp | | Source bytes copied (direct / staged / CBLAS) | 0 / 8192 / 8192 | 0 / 131072 / 131072 | | CBLAS B materialized | 8192 B | 131072 B | | FP32 accumulator (C) | 16384 B | 262144 B | -| End-to-end warm (resolve + direct, B reused) | 6386 ns | 147.5 µs | -| End-to-end cold (bind + prepare + resolve + direct) | 16.4 µs | 265.2 µs | -| Warm repetitions (100, per run, one prepare) | 7146 ns | 150.0 µs | -| INT8 resolve (u8 lanes, one check) | 1218 ns | 1967 ns | -| INT8 stage A (region: one copy per lane) | 307 ns · 4096 B | 3589 ns · 65536 B | -| INT8 stage A, per-value walk (baseline) | 3854 ns | 24.6 µs | -| INT8 B for a region, in place from the table | 35 ns · 0 B written | 30 ns · 0 B written | -| INT8 materialize B (row-major i8) | 7387 ns · 4096 B | 102.3 µs · 65536 B | -| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3971 ns · 66.0 GMAC/s · 3 allocs · 3072 B tmp | 78.8 µs · 212.8 GMAC/s · 3 allocs · 12288 B tmp | -| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 1911 ns · 48% of the GEMM row | 29.5 µs · 37% of the GEMM row | -| INT8 end-to-end warm (resolve + stage + GEMM) | 5984 ns | 86.9 µs | -| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 5003 ns · 4096 B | 79.1 µs · 65536 B | -| INT8 direct GEMM (A in place, prepared B) | 1020 ns · 257.0 GMAC/s · 0 allocs · 0 B of A copied | 33.3 µs · 503.5 GMAC/s · 0 allocs · 0 B of A copied | -| INT8 direct end-to-end warm (resolve + GEMM) | 2994 ns | 40.6 µs | +| End-to-end warm (resolve + direct, B reused) | 6414 ns | 145.1 µs | +| End-to-end cold (bind + prepare + resolve + direct) | 20.9 µs | 259.7 µs | +| Warm repetitions (100, per run, one prepare) | 6364 ns | 150.8 µs | +| INT8 resolve (u8 lanes, one check) | 1201 ns | 1999 ns | +| INT8 stage A (region: one copy per lane) | 306 ns · 4096 B | 4646 ns · 65536 B | +| INT8 stage A, per-value walk (baseline) | 3429 ns | 29.0 µs | +| INT8 B for a region, in place from the table | 30 ns · 0 B written | 30 ns · 0 B written | +| INT8 materialize B (row-major i8) | 6258 ns · 4096 B | 102.1 µs · 65536 B | +| INT8 GEMM `backend::gemm_i8` (packs B per call) | 3549 ns · 73.9 GMAC/s · 3 allocs · 3072 B tmp | 76.9 µs · 218.3 GMAC/s · 3 allocs · 12288 B tmp | +| INT8 B band gather + VNNI pack (done inside each gemm_i8 call) | 2105 ns · 59% of the GEMM row | 29.6 µs · 39% of the GEMM row | +| INT8 end-to-end warm (resolve + stage + GEMM) | 6749 ns | 92.0 µs | +| INT8 prepare B once (PackedI8B bands, `ndarray::simd`) | 5382 ns · 4096 B | 79.4 µs · 65536 B | +| INT8 direct GEMM (A in place, prepared B) | 1021 ns · 256.8 GMAC/s · 0 allocs · 0 B of A copied | 45.9 µs · 365.7 GMAC/s · 0 allocs · 0 B of A copied | +| INT8 direct end-to-end warm (resolve + GEMM) | 3036 ns | 49.1 µs | diff --git a/crates/dr-isa-stream-lab/src/bin/shape.rs b/crates/dr-isa-stream-lab/src/bin/shape.rs index 0efb5767f..927672408 100644 --- a/crates/dr-isa-stream-lab/src/bin/shape.rs +++ b/crates/dr-isa-stream-lab/src/bin/shape.rs @@ -557,13 +557,28 @@ fn main() { }; run(&mut a, &mut c); let want = c.clone(); - let lookups = source.lookups(); let t_translate = median_ns(|| { black_box(translate(&source, &globals).unwrap()); }); + // Inverse map reads of ONE preparation, on a fresh TranslatedB. + let fresh = TranslatedB::new(&bt, &source, &tr).unwrap(); + black_box(int8::PreparedI8B::prepare(&obj, &fresh).unwrap()); + let (pack_reads, admit_reads) = (fresh.reads(), fresh.admission_reads()); let t_prepare = median_ns(|| { black_box(int8::PreparedI8B::prepare(&obj, &tb).unwrap()); }); + // A staging alone, and the GEMM alone on the staged A. + let l = int8::resolve(&obj, &world, &socket).unwrap(); + let useful = int8::stage_a(&obj, &l, &mut a).unwrap(); + let staged = a.len(); + let t_stage = median_ns(|| { + black_box(int8::stage_a(&obj, &l, &mut a).unwrap()); + }); + let k_pad = obj.selection().len().div_ceil(int8::K_STEP) * int8::K_STEP; + let t_gemm = median_ns(|| { + ndarray::simd::int8_gemm_packed_strided(&a, k_pad, prep.bands(), &mut c, dim); + }); + assert_eq!(c, want); let t_warm = median_ns(|| run(&mut a, &mut c)); assert_eq!(c, want); let t_per_op = median_ns(|| { @@ -577,14 +592,30 @@ fn main() { }); assert_eq!(c, want); println!( - "G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles): translate once {} ({} lookups) · prepare B once {} · warm run {} (0 lookups) · re-translate per operation {}\n", + "G2L (64 fields x 64 cols, 200 global ids, source in its own order, 16 tiles, {}):", + ndarray::simd::int8_tile_gemm_tier() + ); + println!("| Step | Time | Movement |\n|---|---|---|"); + println!( + "| translate once | {} | {} G2L lookups |", fmt(t_translate), + tr.lookups() + ); + println!("| prepare B once (admit + pack) | {} | {} admission map reads, {} packing map reads, {} B bytes packed |", fmt(t_prepare), admit_reads, pack_reads, prep.bytes()); + println!( + "| stage A (per warm run) | {} | {} useful bytes copied into a {}-byte padded buffer |", + fmt(t_stage), + useful, + staged + ); + println!("| GEMM only | {} | — |", fmt(t_gemm)); + println!("| warm run (resolve + stage A + GEMM) | {} | 0 G2L lookups, 0 map reads, {} A bytes staged |", fmt(t_warm), useful); + println!( + "| re-translate per operation | {} | {} lookups + {} map reads + B pack + A stage |\n", + fmt(t_per_op), tr.lookups(), - fmt(t_prepare), - fmt(t_warm), - fmt(t_per_op) + admit_reads + pack_reads ); - let _ = lookups; } println!("| Metric | 64×64 | 256×256 |\n|---|---|---|"); diff --git a/crates/dr-isa-stream-lab/src/int8.rs b/crates/dr-isa-stream-lab/src/int8.rs index 6c826dd24..d8078b227 100644 --- a/crates/dr-isa-stream-lab/src/int8.rs +++ b/crates/dr-isa-stream-lab/src/int8.rs @@ -423,6 +423,11 @@ impl PreparedI8B { }) } + /// The packed bands, for timing the GEMM alone. + pub fn bands(&self) -> &[PackedI8B] { + &self.bands + } + /// Bytes the bands occupy, K padding included. pub fn bytes(&self) -> usize { self.bands.iter().map(|b| b.data().len()).sum() From 2969164b04722cd30ec6948f24cfc8033d88ba1f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:46:51 +0000 Subject: [PATCH 8/9] board: regenerate entries index after merging main Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .claude/board/entries/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index fce14731f..c0ba84432 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,7 +25,7 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -593 entries, 2026-08-06 .. 2026-11-10. +594 entries, 2026-08-06 .. 2026-11-10. | date | entry id | finding | file | |---|---|---|---| From 02985218d6052f5ed4427c0fd354b77a541fda3c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 18:55:05 +0000 Subject: [PATCH 9/9] dr-isa-stream-lab: refuse a translation made on another source Two sources built in the same order have equal coordinate digests, so Translation::check admitted a translation from the other one and packing then panicked reading a table with a frame it never issued (Codex, #1994). check now asks the source's own map table whether it issued the translation's frame, and refuses with G2lRefusal::Foreign before packing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/dr-isa-stream-lab/src/g2l.rs | 19 ++++++++++++--- .../tests/integration/g2l.rs | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/crates/dr-isa-stream-lab/src/g2l.rs b/crates/dr-isa-stream-lab/src/g2l.rs index c94104cea..28d81126a 100644 --- a/crates/dr-isa-stream-lab/src/g2l.rs +++ b/crates/dr-isa-stream-lab/src/g2l.rs @@ -38,7 +38,7 @@ use std::cell::Cell; use std::sync::atomic::{AtomicU64, Ordering}; use lance_graph_dt64::dr::{ - Frame, FrameRevisions, LocalMap, LocateRefusal, MapTable, TileId, Width, + Frame, FrameRevisions, LocalMap, LocateRefusal, MapRead, MapTable, TileId, Width, }; use lance_graph_dt64::CoordSpace; @@ -159,6 +159,11 @@ impl OrderedSource { .sum() } + /// The source's coordinate revision: the digest of its current order. + pub fn coordinates(&self) -> u64 { + self.lanes.frames().revisions().coordinates + } + /// Identities held by the source. pub fn len(&self) -> usize { self.order.len() @@ -182,6 +187,9 @@ pub enum G2lRefusal { Duplicate { global: u64 }, /// The translation was made under another order of the source. Stale { made: u64, now: u64 }, + /// The translation was made on another source: its frame was issued by + /// another map table, even if that table holds the same order. + Foreign, } static NEXT_TRANSLATION: AtomicU64 = AtomicU64::new(1); @@ -210,9 +218,14 @@ impl Translation { self.lookups } - /// Refuses once the source's order has moved since the translation was - /// made. One compare. + /// Refuses a translation made on another source (its frame was issued by + /// another map table), and one made under another order of this source. pub fn check(&self, source: &OrderedSource) -> Result<(), G2lRefusal> { + if let Some(&tile) = self.tiles.first() { + if let Err(MapRead::Stale(_)) = source.map.map(&self.frame, tile) { + return Err(G2lRefusal::Foreign); + } + } let now = source.lanes.frames().revisions().coordinates; if now != self.coordinates { return Err(G2lRefusal::Stale { diff --git a/crates/dr-isa-stream-lab/tests/integration/g2l.rs b/crates/dr-isa-stream-lab/tests/integration/g2l.rs index 88ae35fb5..c554d8986 100644 --- a/crates/dr-isa-stream-lab/tests/integration/g2l.rs +++ b/crates/dr-isa-stream-lab/tests/integration/g2l.rs @@ -241,3 +241,26 @@ fn a_translated_identity_past_the_inner_table_is_refused_before_packing() { let tb = TranslatedB::new(&short, &source, &tr).unwrap(); int8::PreparedI8B::prepare(&obj, &tb).expect("every identity is inside the table"); } + +/// A translation belongs to the source whose map table issued its frame. +/// Two sources built in the same order have equal coordinate digests, so a +/// coordinate check alone admits a translation from the other one; packing +/// would then read the second table with a frame it never issued. It must be +/// refused before any packing. +#[test] +fn a_translation_from_another_source_in_the_same_order_is_refused() { + let fx = fx(); + let first = OrderedSource::new(&fx, DIM, order(1237, 11)); + let second = OrderedSource::new(&fx, DIM, order(1237, 11)); + let bt = BTable::new(&B, N, DIM); + let tr = translate(&first, &wanted()).unwrap(); + // Anti-vacuity: the two sources really agree on coordinates. + assert_eq!(first.coordinates(), second.coordinates()); + assert!(matches!( + TranslatedB::new(&bt, &second, &tr), + Err(G2lRefusal::Foreign) + )); + assert!(matches!(tr.check(&second), Err(G2lRefusal::Foreign))); + // Silence twin: the issuing source still admits it. + assert!(TranslatedB::new(&bt, &first, &tr).is_ok()); +}