From 29c16b1c712ab006cd0e035516d664ee04a6cd03 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 8 Nov 2026 00:23:37 +0000 Subject: [PATCH 1/3] Add the DR G2L x ISA stream lab One DT64 lane binding consumed by two computation geometries over the same source-owned bf16 lanes: ARM A folds 16 WHERE masks over a 256x256 identity cube, ARM B computes the same 16x16 SUM(lane) WHERE mask answer as a BF16 tile GEMM (AMX TDPBF16PS on this host), with A read in place from the slab when 16 demanded lanes are consecutive rows at a padded pitch, and staged otherwise. B is the selection, staged from the mask words once per mask set. Tests: both arms exact against a fixture oracle at n = 256, 513, 4096 and 65536; refusals for a lane in another row order, a stale frame, masks in another space, scattered lanes, permuted rows, an unpadded pitch and a partial row tile; bf16 exactness. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- Cargo.toml | 5 + crates/dr-isa-stream-lab/Cargo.toml | 27 ++ crates/dr-isa-stream-lab/src/bin/bench.rs | 148 ++++++++ crates/dr-isa-stream-lab/src/bind.rs | 121 +++++++ crates/dr-isa-stream-lab/src/cube.rs | 136 +++++++ crates/dr-isa-stream-lab/src/fixture.rs | 62 ++++ crates/dr-isa-stream-lab/src/gemm.rs | 331 ++++++++++++++++++ crates/dr-isa-stream-lab/src/lib.rs | 24 ++ crates/dr-isa-stream-lab/src/oracle.rs | 23 ++ crates/dr-isa-stream-lab/src/source.rs | 240 +++++++++++++ .../tests/integration/geometry.rs | 182 ++++++++++ .../tests/integration/main.rs | 4 + .../tests/integration/numeric.rs | 28 ++ .../tests/integration/parity.rs | 135 +++++++ .../tests/integration/refusals.rs | 129 +++++++ 15 files changed, 1595 insertions(+) create mode 100644 crates/dr-isa-stream-lab/Cargo.toml create mode 100644 crates/dr-isa-stream-lab/src/bin/bench.rs create mode 100644 crates/dr-isa-stream-lab/src/bind.rs create mode 100644 crates/dr-isa-stream-lab/src/cube.rs create mode 100644 crates/dr-isa-stream-lab/src/fixture.rs create mode 100644 crates/dr-isa-stream-lab/src/gemm.rs create mode 100644 crates/dr-isa-stream-lab/src/lib.rs create mode 100644 crates/dr-isa-stream-lab/src/oracle.rs create mode 100644 crates/dr-isa-stream-lab/src/source.rs create mode 100644 crates/dr-isa-stream-lab/tests/integration/geometry.rs create mode 100644 crates/dr-isa-stream-lab/tests/integration/main.rs create mode 100644 crates/dr-isa-stream-lab/tests/integration/numeric.rs create mode 100644 crates/dr-isa-stream-lab/tests/integration/parity.rs create mode 100644 crates/dr-isa-stream-lab/tests/integration/refusals.rs diff --git a/Cargo.toml b/Cargo.toml index 0cd45ec4c..214702aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -225,6 +225,11 @@ exclude = [ # socket as coordinate witness. Own workspace root. Verify via # `cargo test --release --manifest-path crates/soa64-u64-lab/Cargo.toml`. "crates/soa64-u64-lab", + # DR G2L x ISA stream lab: one DT64 lane binding feeding a 16-mask cube + # fold and a BF16 tile GEMM (AMX where present). Own workspace root; needs + # ndarray's strided BF16 GEMM entry. Verify via + # `cargo test --release --manifest-path crates/dr-isa-stream-lab/Cargo.toml`. + "crates/dr-isa-stream-lab", # ERA5 0.25-degree global grid baked onto the canonical 512-byte NodeRow -- # one cell = one node, one timestep = one Lance version. Standalone and # genuinely zero-dep (see the crate manifest for why helix is NOT an diff --git a/crates/dr-isa-stream-lab/Cargo.toml b/crates/dr-isa-stream-lab/Cargo.toml new file mode 100644 index 000000000..66a560f13 --- /dev/null +++ b/crates/dr-isa-stream-lab/Cargo.toml @@ -0,0 +1,27 @@ +# DR G2L x ISA stream lab: one DT64 lane binding consumed by two computation +# geometries -- a positional 16-mask fold over a 256x256 identity cube, and a +# BF16 tile GEMM (AMX TDPBF16PS where the host has it). Own workspace root; +# nothing depends on it. Needs ndarray with +# `bf16_tile_gemm_16x16_packed_strided` (AdaWorldAPI/ndarray#414). +# cargo test --release --manifest-path crates/dr-isa-stream-lab/Cargo.toml +# cargo run --release --manifest-path crates/dr-isa-stream-lab/Cargo.toml --bin bench +[package] +name = "dr-isa-stream-lab" +version = "0.0.0" +edition = "2021" +publish = false +license = "MIT OR Apache-2.0" + +[workspace] + +[dependencies] +lance-graph-dt64 = { path = "../lance-graph-dt64" } +lance-graph-contract = { path = "../lance-graph-contract" } +ndarray = { path = "../../../ndarray", default-features = false, features = ["std"] } + +[[test]] +name = "integration" +path = "tests/integration/main.rs" + +[profile.release] +debug = false diff --git a/crates/dr-isa-stream-lab/src/bin/bench.rs b/crates/dr-isa-stream-lab/src/bin/bench.rs new file mode 100644 index 000000000..45435e26c --- /dev/null +++ b/crates/dr-isa-stream-lab/src/bin/bench.rs @@ -0,0 +1,148 @@ +//! Stage costs for one binding feeding both arms. Every timed result is +//! checked against the fixture oracle; a mismatch aborts the run, because a +//! fast wrong answer is not a measurement (AMX tile state can corrupt under +//! host contention, ndarray `.claude/AMX_GOTCHAS.md` Gotcha 14). +//! +//! Units: ns per operation, us per batch. Rows marked "reference" are +//! oracle-grade paths kept for comparison; they are not gates. + +use std::hint::black_box; +use std::time::Instant; + +use dr_isa_stream_lab::bind::{World, SPACE_A}; +use dr_isa_stream_lab::cube::{fold, WhereMasks}; +use dr_isa_stream_lab::fixture::{Fixture, MASKS}; +use dr_isa_stream_lab::gemm::{sums, sums_whole, Mode, SelectionB}; +use dr_isa_stream_lab::oracle; +use dr_isa_stream_lab::source::{Bf16Lanes, Socket}; + +const RUNS: usize = 41; + +fn median_ns(mut f: impl FnMut()) -> u128 { + f(); + let mut t: Vec = (0..RUNS) + .map(|_| { + let s = Instant::now(); + f(); + s.elapsed().as_nanos() + }) + .collect(); + t.sort_unstable(); + t[RUNS / 2] +} + +fn us(ns: u128) -> f64 { + ns as f64 / 1000.0 +} + +fn main() { + println!("tier: {}", ndarray::simd::bf16_tile_gemm_tier()); + for (n, props) in [(65_536usize, 16usize), (65_536, 32)] { + let fx = Fixture { + n, + props, + seed: 0xBE7C, + }; + let fields: Vec = (0..props as u8).collect(); + let want = oracle::sums(&fx, &(0..props).collect::>()); + let want_f: Vec<[f32; MASKS]> = want.iter().map(|r| r.map(|v| v as f32)).collect(); + + let slab = Bf16Lanes::slab_in_order(&fx).unwrap(); + let scattered = Bf16Lanes::scattered(&fx).unwrap(); + let world = World::new(); + let s_slab = Socket::in_order(&slab, props); + let s_scat = Socket::in_order(&scattered, props); + + let bind_ns = median_ns(|| { + black_box(world.bind(&s_slab, &fields, SPACE_A).unwrap()); + }); + let binding = world.bind(&s_slab, &fields, SPACE_A).unwrap(); + let b_scat = world.bind(&s_scat, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let counts_ns = median_ns(|| { + black_box(masks.counts()); + }); + let stage_ns = median_ns(|| { + black_box(SelectionB::stage(&masks)); + }); + let sel = SelectionB::stage(&masks); + + let check = |label: &str, run: &dyn Fn() -> (Vec<[f32; MASKS]>, usize, usize)| { + let (got, gemms, staged) = run(); + assert_eq!(got, want_f, "{label}: wrong answer"); + let ns = median_ns(|| { + let (got, _, _) = run(); + assert_eq!(got, want_f, "{label}: wrong answer while timing"); + black_box(got); + }); + (ns, gemms, staged) + }; + let whole = |b: &_, s: &Socket<'_>, mode| { + let (got, r) = sums_whole(&world, b, s, &sel, &fields, mode).unwrap(); + (got, r.gemms, r.a_staged_bytes) + }; + let blocked = |mode| { + let (got, r) = sums(&world, &binding, &s_slab, &masks, &fields, mode).unwrap(); + (got, r.gemms, r.a_staged_bytes) + }; + let wd = check("whole direct", &|| whole(&binding, &s_slab, Mode::Direct)); + let ws = check("whole staged slab", &|| { + whole(&binding, &s_slab, Mode::Staged) + }); + let wc = check("whole staged scattered", &|| { + whole(&b_scat, &s_scat, Mode::Staged) + }); + let bd = check("blocked direct", &|| blocked(Mode::Direct)); + + assert_eq!( + fold(&world, &binding, &s_slab, &masks, &fields).unwrap(), + want + ); + let fold_ns = median_ns(|| { + black_box(fold(&world, &binding, &s_slab, &masks, &fields).unwrap()); + }); + + println!( + "\nn={n} props={props} masks={MASKS} selected={} lanes={} B selection={} B", + masks.counts().iter().sum::(), + slab.resident_bytes(), + sel.bytes() + ); + println!(" bind, cold, once {bind_ns:>9} ns"); + println!( + " ARM A: 16 WHERE counts (popcount) {:>9.1} us", + us(counts_ns) + ); + println!( + " stage selection B, cold, per masks {:>9.1} us", + us(stage_ns) + ); + println!( + " ARM B whole lane, A in place {:>9.1} us gemms={} A-staged={} B", + us(wd.0), + wd.1, + wd.2 + ); + println!( + " ARM B whole lane, A staged (slab) {:>9.1} us gemms={} A-staged={} B", + us(ws.0), + ws.1, + ws.2 + ); + println!( + " ARM B whole lane, A staged (scatter) {:>9.1} us gemms={} A-staged={} B", + us(wc.0), + wc.1, + wc.2 + ); + println!( + " reference: ARM B per-32-block calls {:>9.1} us gemms={} (B restaged per block)", + us(bd.0), + bd.1 + ); + println!( + " reference: ARM A per-bit sum fold {:>9.1} us", + us(fold_ns) + ); + } +} diff --git a/crates/dr-isa-stream-lab/src/bind.rs b/crates/dr-isa-stream-lab/src/bind.rs new file mode 100644 index 000000000..680b108c7 --- /dev/null +++ b/crates/dr-isa-stream-lab/src/bind.rs @@ -0,0 +1,121 @@ +//! The class, its view and route, and binding demanded fields through DT64. + +use lance_graph_contract::class_view::{ClassId, ClassView, WideFieldMask}; +use lance_graph_contract::ontology::{DisplayTemplate, FieldRef}; +use lance_graph_dt64::dr::Scope; +use lance_graph_dt64::{ + BindStore, ClassSchema, CoordSpace, Extent, LaneBinding, LaneRefusal, LaneRequest, LaneSource, + RouteAt, SchemaRev, +}; + +use crate::source::Socket; + +/// The lab's class. +pub const CLASS: u32 = 0x5115_0001; +/// The ClassView key of the class. +pub const VIEW_CLASS: ClassId = 0x5115; +/// DR position the class is routed at. +pub const POSITION: u16 = 5; +/// The reader scope every lane serves. +pub const READER: Scope = Scope { id: 9 }; +/// The row order of the canonical population. +pub const SPACE_A: CoordSpace = CoordSpace { + population: 1, + coordinates: 1, +}; +/// Another row order of the same population. +pub const SPACE_OTHER: CoordSpace = CoordSpace { + population: 1, + coordinates: 2, +}; +/// Fields of the class. +pub const FIELDS: usize = 32; + +/// A ClassView: field `f` is `isa:f{f}`. +pub struct View(Vec); + +impl ClassView for View { + fn fields(&self, class: ClassId) -> &[FieldRef] { + if class == VIEW_CLASS { + &self.0 + } else { + &[] + } + } + fn template(&self, _class: ClassId) -> DisplayTemplate { + DisplayTemplate::Detail + } + fn dolce_category_id(&self, _class: ClassId) -> u8 { + 0 + } +} + +/// Store, view and route of the class. +pub struct World { + pub store: BindStore, + pub view: View, + pub route: RouteAt, +} + +impl World { + pub fn new() -> World { + let mut store = BindStore::new(); + let view = View( + (0..FIELDS) + .map(|f| FieldRef::new(format!("isa:f{f}"), format!("F{f}"))) + .collect(), + ); + store + .register( + &view, + CLASS, + ClassSchema { + schema_rev: SchemaRev(1), + field_count: FIELDS, + extent: Extent::DR64_CUBE, + }, + Vec::new(), + ) + .expect("the lab class registers"); + let version = store.route(POSITION, CLASS).expect("the lab class routes"); + World { + store, + view, + route: RouteAt { + position: POSITION, + version, + }, + } + } + + /// Cold: bind exactly `fields` under `selection`'s row order, at the + /// source's current frame. + pub fn bind( + &self, + socket: &Socket<'_>, + fields: &[u8], + selection: CoordSpace, + ) -> Result { + let mask = WideFieldMask::from_positions(fields); + let frame = socket.frames().seal(); + self.store.bind_lanes( + &self.view, + socket.frames(), + socket, + LaneRequest { + route: self.route, + rev: SchemaRev(1), + fields: &mask, + frame: &frame, + scope: READER, + selection, + }, + ) + } +} + +impl Default for World { + fn default() -> World { + World::new() + } +} diff --git a/crates/dr-isa-stream-lab/src/cube.rs b/crates/dr-isa-stream-lab/src/cube.rs new file mode 100644 index 000000000..625d4a438 --- /dev/null +++ b/crates/dr-isa-stream-lab/src/cube.rs @@ -0,0 +1,136 @@ +//! ARM A: 16 WHERE masks over a 256 x 256 identity cube, folded in place. +//! +//! Identity `i` sits at `tile(i) = i >> 8`, `slot(i) = i & 255`. A mask is one +//! bit per identity in the binding's row order, so 16 masks over 65,536 +//! identities are 16 x 1,024 words. The fold reads each demanded lane once, +//! at the positions the mask words name; nothing is copied out. + +use lance_graph_dt64::{CoordSpace, LaneBinding, LaneRefusal}; + +use crate::bind::World; +use crate::fixture::{bf16_value, Fixture, MASKS}; +use crate::source::Socket; + +/// Identities per cube tile. +pub const TILE: usize = 256; + +/// Cube tile of identity `i`. +pub const fn tile(i: usize) -> usize { + i >> 8 +} + +/// Slot of identity `i` within its tile. +pub const fn slot(i: usize) -> usize { + i & 255 +} + +/// The 16 WHERE masks, in one coordinate space. +pub struct WhereMasks { + pub space: CoordSpace, + pub n: usize, + words: Vec>, +} + +impl WhereMasks { + /// The fixture's masks, declared to be in `space`. + pub fn from_fixture(fx: &Fixture, space: CoordSpace) -> WhereMasks { + let len = fx.n.div_ceil(64); + let words = (0..MASKS) + .map(|m| { + let mut w = vec![0u64; len].into_boxed_slice(); + for i in 0..fx.n { + if fx.selected(i, m) { + w[i / 64] |= 1 << (i % 64); + } + } + w + }) + .collect(); + WhereMasks { + space, + n: fx.n, + words, + } + } + + /// Mask `m`'s words. + pub fn words(&self, m: usize) -> &[u64] { + &self.words[m] + } + + /// Selected identities per mask (a popcount terminal). + pub fn counts(&self) -> [u32; MASKS] { + let mut c = [0u32; MASKS]; + for (m, slot) in c.iter_mut().enumerate() { + *slot = self.words[m].iter().map(|w| w.count_ones()).sum(); + } + c + } +} + +/// Why an arm refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ArmRefusal { + /// The binding refused (frame, space, scope, route, ...). + Lane(LaneRefusal), + /// The masks are in another row order than the binding's selection. + MaskSpace { + masks: CoordSpace, + selection: CoordSpace, + }, +} + +impl From for ArmRefusal { + fn from(r: LaneRefusal) -> Self { + ArmRefusal::Lane(r) + } +} + +/// Refuse masks whose row order is not the binding's. +pub fn check_masks(binding: &LaneBinding, masks: &WhereMasks) -> Result<(), ArmRefusal> { + if masks.space == binding.selection() { + Ok(()) + } else { + Err(ArmRefusal::MaskSpace { + masks: masks.space, + selection: binding.selection(), + }) + } +} + +/// `SUM(field) WHERE mask m` for every listed field and every mask, by a +/// positional fold over the cube: tile by tile, each mask's four words name +/// the selected slots, and the lane is read at exactly those positions. +pub fn fold( + world: &World, + binding: &LaneBinding, + socket: &Socket<'_>, + masks: &WhereMasks, + fields: &[u8], +) -> Result, ArmRefusal> { + check_masks(binding, masks)?; + let mut out = Vec::with_capacity(fields.len()); + for &f in fields { + let lane = binding.lane(&world.store, &world.view, socket.lanes.frames(), socket, f)?; + let values = lane.values(); + let mut row = [0i64; MASKS]; + for t in 0..masks.n.div_ceil(TILE) { + for (m, slot) in row.iter_mut().enumerate() { + let words = masks.words(m); + let lo = t * 4; + let hi = (lo + 4).min(words.len()); + for (w, &word) in words[lo..hi].iter().enumerate() { + let w = lo + w; + let mut bits = word; + while bits != 0 { + let i = w * 64 + bits.trailing_zeros() as usize; + *slot += bf16_value(values[i]) as i64; + bits &= bits - 1; + } + } + } + } + out.push(row); + } + Ok(out) +} diff --git a/crates/dr-isa-stream-lab/src/fixture.rs b/crates/dr-isa-stream-lab/src/fixture.rs new file mode 100644 index 000000000..c7f51ad60 --- /dev/null +++ b/crates/dr-isa-stream-lab/src/fixture.rs @@ -0,0 +1,62 @@ +//! Deterministic values and WHERE masks. Nothing else defines them. + +/// Number of WHERE masks, and the GEMM's N. +pub const MASKS: usize = 16; +/// Property lanes per GEMM row tile (the GEMM's M). +pub const ROWS: usize = 16; +/// Identities per GEMM K block. +pub const BLOCK: usize = 32; + +/// The value set: an identity count, a property count and a seed. +#[derive(Debug, Clone, Copy)] +pub struct Fixture { + pub n: usize, + pub props: usize, + pub seed: u64, +} + +fn mix(mut x: u64) -> u64 { + x ^= x >> 33; + x = x.wrapping_mul(0xff51_afd7_ed55_8ccd); + x ^= x >> 33; + x = x.wrapping_mul(0xc4ce_b9fe_1a85_ec53); + x ^ (x >> 33) +} + +impl Fixture { + /// Property `p` of identity `i`: an integer in [-8, 8], exact in bf16. + pub fn value(&self, i: usize, p: usize) -> i32 { + let h = mix(self.seed ^ ((p as u64) << 40) ^ i as u64); + (h % 17) as i32 - 8 + } + + /// Whether identity `i` is selected by WHERE mask `m`. + pub fn selected(&self, i: usize, m: usize) -> bool { + // Mask m keeps roughly (m + 1) / 17 of the population, so the 16 + // masks differ in density as well as membership. + mix(self.seed.rotate_left(17) ^ ((m as u64) << 48) ^ i as u64) % 17 <= m as u64 + } +} + +/// bf16 encoding of `v`, or `None` when bf16 cannot hold it exactly. +pub fn bf16_exact(v: i32) -> Option { + let f = v as f32; + let bits = f.to_bits(); + if bits & 0xFFFF != 0 || f as i32 != v { + None + } else { + Some((bits >> 16) as u16) + } +} + +/// bf16 encoding of `v` by round-to-nearest-even; approximate by design. +pub fn bf16_round(v: f32) -> u16 { + let bits = v.to_bits(); + let lsb = (bits >> 16) & 1; + ((bits + 0x7FFF + lsb) >> 16) as u16 +} + +/// The f32 value of a bf16 encoding. +pub fn bf16_value(h: u16) -> f32 { + f32::from_bits(u32::from(h) << 16) +} diff --git a/crates/dr-isa-stream-lab/src/gemm.rs b/crates/dr-isa-stream-lab/src/gemm.rs new file mode 100644 index 000000000..7f55e344c --- /dev/null +++ b/crates/dr-isa-stream-lab/src/gemm.rs @@ -0,0 +1,331 @@ +//! ARM B: the cube's 16 x 16 answer as a BF16 tile GEMM. +//! +//! `C[p][m] = sum_i lane_p[i] * mask_m(i)`: M = 16 demanded lanes, N = 16 +//! masks, K = identities in blocks of 32. B is the selection, a 0/1 matrix +//! staged straight into the VNNI layout from the mask words; it is derived +//! from the WHERE result, never source data. +//! +//! A is the source's lanes. When 16 demanded lanes are consecutive rows of +//! one slab, A is read in place through +//! `bf16_tile_gemm_16x16_packed_strided`, with the slab pitch as the row +//! stride. Otherwise each block's 16 x 32 A is copied into a staging tile, +//! and [`Report::a_staged_bytes`] says how much was copied. + +use lance_graph_dt64::{LaneBinding, LaneRefusal}; +use ndarray::simd::{ + bf16_tile_gemm_16x16_packed, bf16_tile_gemm_16x16_packed_strided, bf16_tile_gemm_tier, + PackedBf16B, +}; + +use crate::bind::World; +use crate::cube::{check_masks, ArmRefusal, WhereMasks}; +use crate::fixture::{BLOCK, MASKS, ROWS}; +use crate::source::{SlabLane, Socket}; + +/// bf16 1.0. +const ONE: u16 = 0x3F80; + +/// How A reaches the tile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// In place from the slab; refuse if the geometry does not allow it. + Direct, + /// Always through a staging tile. + Staged, +} + +/// Why A cannot be read in place. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DirectRefusal { + /// Fewer than 16 demanded lanes in this row tile. + PartialRowTile { lanes: usize }, + /// The lane is not in the same allocation as the first lane of its tile. + NotOneSlab { field: u8 }, + /// The lane's pitch differs from the first lane's. + PitchMismatch { field: u8 }, + /// The lane is not the row after the previous lane. + RowsNotConsecutive { field: u8 }, + /// The pitch does not cover the last block with padding: a 32-wide read + /// would run into the next row instead of zeros. + ShortPitch { pitch: usize, needed: usize }, +} + +/// Why ARM B refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum GemmRefusal { + Arm(ArmRefusal), + Direct(DirectRefusal), +} + +impl From for GemmRefusal { + fn from(r: ArmRefusal) -> Self { + GemmRefusal::Arm(r) + } +} + +impl From for GemmRefusal { + fn from(r: LaneRefusal) -> Self { + GemmRefusal::Arm(ArmRefusal::Lane(r)) + } +} + +/// What ARM B did. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Report { + /// The GEMM tier that ran. + pub tier: &'static str, + /// Row tiles read in place. + pub direct_tiles: usize, + /// Row tiles staged. + pub staged_tiles: usize, + /// Bytes of A copied into staging tiles. + pub a_staged_bytes: usize, + /// Nonzero B entries written (each is written once and cleared once). + pub b_entries: usize, + /// GEMM calls. + pub gemms: usize, +} + +/// In-place geometry of one row tile: A's row `r` is +/// `slab[(first_row + r) * pitch..]`. +struct Geometry<'a> { + slab: &'a [u16], + first_row: usize, + pitch: usize, +} + +fn geometry<'a>(fields: &[u8], lanes: &[SlabLane<'a>]) -> Result, DirectRefusal> { + if lanes.len() != ROWS { + return Err(DirectRefusal::PartialRowTile { lanes: lanes.len() }); + } + let first = lanes[0]; + for (k, (lane, &field)) in lanes.iter().zip(fields).enumerate().skip(1) { + if !std::ptr::eq(lane.slab, first.slab) { + return Err(DirectRefusal::NotOneSlab { field }); + } + if lane.pitch != first.pitch { + return Err(DirectRefusal::PitchMismatch { field }); + } + if lane.row != first.row + k { + return Err(DirectRefusal::RowsNotConsecutive { field }); + } + } + let needed = first.n.div_ceil(BLOCK) * BLOCK; + if first.pitch < needed { + return Err(DirectRefusal::ShortPitch { + pitch: first.pitch, + needed, + }); + } + Ok(Geometry { + slab: first.slab, + first_row: first.row, + pitch: first.pitch, + }) +} + +/// Stage block `t` of the selection into `b`: `B[j][m] = mask_m(32t + j)`. +/// Returns the number of entries written. +fn stage_b(masks: &WhereMasks, t: usize, b: &mut PackedBf16B, value: u16) -> usize { + let mut written = 0; + for m in 0..MASKS { + let word = masks.words(m)[t / 2]; + let mut bits = (word >> ((t % 2) * 32)) & 0xFFFF_FFFF; + while bits != 0 { + let j = bits.trailing_zeros() as usize; + b.data_mut()[PackedBf16B::vnni_index(j, m)] = value; + written += 1; + bits &= bits - 1; + } + } + written +} + +/// `SUM(field) WHERE mask m` for every listed field and mask, as tile GEMMs. +/// Fields are taken 16 at a time; a final partial group is staged. +pub fn sums( + world: &World, + binding: &LaneBinding, + socket: &Socket<'_>, + masks: &WhereMasks, + fields: &[u8], + mode: Mode, +) -> Result<(Vec<[f32; MASKS]>, Report), GemmRefusal> { + check_masks(binding, masks)?; + let mut lanes = Vec::with_capacity(fields.len()); + for &f in fields { + lanes.push(binding.lane(&world.store, &world.view, socket.lanes.frames(), socket, f)?); + } + let groups: Vec<(&[u8], &[SlabLane<'_>])> = + fields.chunks(ROWS).zip(lanes.chunks(ROWS)).collect(); + let mut plans = Vec::with_capacity(groups.len()); + for (fs, ls) in &groups { + plans.push(match (mode, geometry(fs, ls)) { + (Mode::Direct, Ok(g)) => Some(g), + (Mode::Direct, Err(e)) => return Err(GemmRefusal::Direct(e)), + (Mode::Staged, _) => None, + }); + } + + let mut report = Report { + tier: bf16_tile_gemm_tier(), + ..Report::default() + }; + let mut c = vec![[0f32; ROWS * MASKS]; groups.len()]; + let mut b = PackedBf16B::zeroed(BLOCK); + let mut a = vec![0u16; ROWS * BLOCK]; + for t in 0..masks.n.div_ceil(BLOCK) { + let written = stage_b(masks, t, &mut b, ONE); + report.b_entries += written; + if written != 0 { + for (g, plan) in plans.iter().enumerate() { + match plan { + Some(geo) => { + let start = geo.first_row * geo.pitch + t * BLOCK; + bf16_tile_gemm_16x16_packed_strided( + &geo.slab[start..], + geo.pitch, + &b, + &mut c[g], + ); + } + None => { + a.fill(0); + for (r, lane) in groups[g].1.iter().enumerate() { + let values = lane.values(); + let lo = t * BLOCK; + let hi = (lo + BLOCK).min(values.len()); + a[r * BLOCK..r * BLOCK + (hi - lo)].copy_from_slice(&values[lo..hi]); + report.a_staged_bytes += (hi - lo) * 2; + } + bf16_tile_gemm_16x16_packed(&a, &b, &mut c[g]); + } + } + report.gemms += 1; + } + stage_b(masks, t, &mut b, 0); + } + } + for plan in &plans { + if plan.is_some() { + report.direct_tiles += 1; + } else { + report.staged_tiles += 1; + } + } + let mut out = Vec::with_capacity(fields.len()); + for (g, (fs, _)) in groups.iter().enumerate() { + for r in 0..fs.len() { + let mut row = [0f32; MASKS]; + row.copy_from_slice(&c[g][r * MASKS..(r + 1) * MASKS]); + out.push(row); + } + } + Ok((out, report)) +} + +/// The selection as one prepacked B over the whole population: +/// `B[i][m] = mask_m(i)` for `i < n`, zero to the padded K. It depends only +/// on the masks, so it is staged once and reused by every row tile and every +/// query over the same masks. It is a numeric realization of the WHERE +/// result, tied to the masks' coordinate space; it is never source data. +pub struct SelectionB { + pub space: lance_graph_dt64::CoordSpace, + pub n: usize, + b: PackedBf16B, +} + +impl SelectionB { + /// Stage the masks in VNNI layout: a sequential write of every entry. + pub fn stage(masks: &WhereMasks) -> SelectionB { + let k = masks.n.div_ceil(BLOCK) * BLOCK; + let mut b = PackedBf16B::zeroed(k); + let data = b.data_mut(); + for m in 0..MASKS { + for (w, &word) in masks.words(m).iter().enumerate() { + let mut bits = word; + while bits != 0 { + let i = w * 64 + bits.trailing_zeros() as usize; + data[PackedBf16B::vnni_index(i, m)] = ONE; + bits &= bits - 1; + } + } + } + SelectionB { + space: masks.space, + n: masks.n, + b, + } + } + + /// Bytes the staged selection occupies. + pub fn bytes(&self) -> usize { + self.b.data().len() * 2 + } +} + +/// Like [`sums`], over a prepacked [`SelectionB`]: one GEMM per 16-lane row +/// tile with K = the padded population. In [`Mode::Direct`] A is each tile's +/// 16 lanes read in place at the slab pitch; in [`Mode::Staged`] they are +/// first copied into a dense 16 x K buffer. +pub fn sums_whole( + world: &World, + binding: &LaneBinding, + socket: &Socket<'_>, + sel: &SelectionB, + fields: &[u8], + mode: Mode, +) -> Result<(Vec<[f32; MASKS]>, Report), GemmRefusal> { + if sel.space != binding.selection() { + return Err(GemmRefusal::Arm(ArmRefusal::MaskSpace { + masks: sel.space, + selection: binding.selection(), + })); + } + let k = sel.b.k(); + let mut report = Report { + tier: bf16_tile_gemm_tier(), + ..Report::default() + }; + let mut out = Vec::with_capacity(fields.len()); + let mut dense = Vec::new(); + for fs in fields.chunks(ROWS) { + let mut lanes = Vec::with_capacity(fs.len()); + for &f in fs { + lanes.push(binding.lane( + &world.store, + &world.view, + socket.lanes.frames(), + socket, + f, + )?); + } + let mut c = [0f32; ROWS * MASKS]; + match (mode, geometry(fs, &lanes)) { + (Mode::Direct, Ok(geo)) => { + let start = geo.first_row * geo.pitch; + bf16_tile_gemm_16x16_packed_strided(&geo.slab[start..], geo.pitch, &sel.b, &mut c); + report.direct_tiles += 1; + } + (Mode::Direct, Err(e)) => return Err(GemmRefusal::Direct(e)), + (Mode::Staged, _) => { + dense.clear(); + dense.resize(ROWS * k, 0u16); + for (r, lane) in lanes.iter().enumerate() { + let values = lane.values(); + dense[r * k..r * k + values.len()].copy_from_slice(values); + report.a_staged_bytes += values.len() * 2; + } + bf16_tile_gemm_16x16_packed(&dense, &sel.b, &mut c); + report.staged_tiles += 1; + } + } + report.gemms += 1; + for r in 0..fs.len() { + let mut row = [0f32; MASKS]; + row.copy_from_slice(&c[r * MASKS..(r + 1) * MASKS]); + out.push(row); + } + } + Ok((out, report)) +} diff --git a/crates/dr-isa-stream-lab/src/lib.rs b/crates/dr-isa-stream-lab/src/lib.rs new file mode 100644 index 000000000..db745c729 --- /dev/null +++ b/crates/dr-isa-stream-lab/src/lib.rs @@ -0,0 +1,24 @@ +//! DR G2L x ISA stream lab. +//! +//! The question: can one source-witnessed DT64 lane binding feed two very +//! different computations without a second value layout? +//! +//! * [`fixture`] is the only source of truth for values and WHERE masks. +//! * [`source`] owns the lanes: one bf16 slab with a uniform pitch, or +//! separately allocated lanes. It implements DT64's [`LaneSource`]. +//! * [`bind`] binds demanded fields through the DT64 lane socket. +//! * [`cube`] is ARM A: 16 WHERE masks over a 256 x 256 identity cube, folded +//! positionally into `SUM(lane p) WHERE mask m`. +//! * [`gemm`] is ARM B: the same 16 x 16 answer as a BF16 tile GEMM, +//! `C[p][m] = sum_i lane_p[i] * mask_m(i)`, with A read in place from the +//! slab when its geometry allows and staged otherwise. +//! * [`oracle`] computes the answer from the fixture alone. +//! +//! [`LaneSource`]: lance_graph_dt64::LaneSource + +pub mod bind; +pub mod cube; +pub mod fixture; +pub mod gemm; +pub mod oracle; +pub mod source; diff --git a/crates/dr-isa-stream-lab/src/oracle.rs b/crates/dr-isa-stream-lab/src/oracle.rs new file mode 100644 index 000000000..6b12b06fe --- /dev/null +++ b/crates/dr-isa-stream-lab/src/oracle.rs @@ -0,0 +1,23 @@ +//! The answer straight from the fixture: no lanes, no masks, no binding. + +use crate::fixture::{Fixture, MASKS}; + +/// `answer[p][m] = sum of value(i, p) over identities selected by mask m`, +/// for the listed properties. +pub fn sums(fx: &Fixture, props: &[usize]) -> Vec<[i64; MASKS]> { + props + .iter() + .map(|&p| { + let mut row = [0i64; MASKS]; + for i in 0..fx.n { + let v = i64::from(fx.value(i, p)); + for (m, slot) in row.iter_mut().enumerate() { + if fx.selected(i, m) { + *slot += v; + } + } + } + row + }) + .collect() +} diff --git a/crates/dr-isa-stream-lab/src/source.rs b/crates/dr-isa-stream-lab/src/source.rs new file mode 100644 index 000000000..b3bbfe5a0 --- /dev/null +++ b/crates/dr-isa-stream-lab/src/source.rs @@ -0,0 +1,240 @@ +//! Source-owned bf16 lanes and the DT64 socket over them. +//! +//! A [`SlabLane`] is a borrowed view that says where the lane sits: which +//! allocation, which row, at which pitch. DT64 hands out lane addresses; the +//! physical geometry stays with the source and its lane type, so DR carries +//! no ISA knowledge. + +use std::cell::Cell; + +use lance_graph_contract::ontology::FieldRef; +use lance_graph_dt64::dr::{Frame, FrameRevisions, FrameStore, Scope}; +use lance_graph_dt64::{CoordSpace, LaneAddr, LaneSource, SchemaRev}; + +use crate::bind::{CLASS, READER, SPACE_A}; +use crate::fixture::{bf16_exact, Fixture, BLOCK}; + +/// One borrowed lane: `n` bf16 values at `slab[row * pitch..][..n]`. +#[derive(Debug, Clone, Copy)] +pub struct SlabLane<'a> { + pub slab: &'a [u16], + pub row: usize, + pub pitch: usize, + pub n: usize, +} + +impl<'a> SlabLane<'a> { + /// The lane's values, in the lane's own row order. + pub fn values(&self) -> &'a [u16] { + &self.slab[self.row * self.pitch..self.row * self.pitch + self.n] + } +} + +/// Why a source could not be built from the fixture. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IngestRefusal { + /// The value cannot be held exactly in bf16. + NotBf16Exact { + identity: usize, + prop: usize, + value: i32, + }, +} + +fn frames() -> FrameStore { + FrameStore::new(FrameRevisions { + source: 1, + relation_map: 1, + coordinates: 1, + }) +} + +/// How a source lays out its lanes. +#[derive(Debug)] +enum Layout { + /// One allocation; lane `r` at `data[r * pitch..]`, `pitch >= n` and a + /// multiple of [`BLOCK`], padding zeroed. + Slab { data: Box<[u16]>, pitch: usize }, + /// One allocation per lane. + Scattered(Vec>), +} + +/// The data owner: bf16 lanes, the frame store that lives with them, and the +/// field-to-row map (`rows[f]` is the physical row holding field `f`). +#[derive(Debug)] +pub struct Bf16Lanes { + layout: Layout, + n: usize, + frames: FrameStore, +} + +impl Bf16Lanes { + /// Lanes in one slab, field `f` stored at physical row `rows[f]`. + pub fn slab(fx: &Fixture, rows: &[usize]) -> Result { + Bf16Lanes::slab_with_pitch(fx, rows, fx.n.div_ceil(BLOCK) * BLOCK) + } + + /// Lanes in one slab at an explicit `pitch >= n`, with no guarantee that + /// the pitch is padded to a [`BLOCK`] multiple. + pub fn slab_with_pitch( + fx: &Fixture, + rows: &[usize], + pitch: usize, + ) -> Result { + assert!(pitch >= fx.n, "pitch {pitch} shorter than n {}", fx.n); + let physical = rows.iter().max().map_or(0, |&r| r + 1); + let mut data = vec![0u16; physical * pitch].into_boxed_slice(); + for (p, &r) in rows.iter().enumerate() { + fill(fx, p, &mut data[r * pitch..r * pitch + fx.n])?; + } + Ok(Bf16Lanes { + layout: Layout::Slab { data, pitch }, + n: fx.n, + frames: frames(), + }) + } + + /// Lanes in one slab, field `f` at row `f`. + pub fn slab_in_order(fx: &Fixture) -> Result { + let rows: Vec = (0..fx.props).collect(); + Bf16Lanes::slab(fx, &rows) + } + + /// Every lane in its own allocation. + pub fn scattered(fx: &Fixture) -> Result { + let mut lanes = Vec::with_capacity(fx.props); + for p in 0..fx.props { + let mut lane = vec![0u16; fx.n].into_boxed_slice(); + fill(fx, p, &mut lane)?; + lanes.push(lane); + } + Ok(Bf16Lanes { + layout: Layout::Scattered(lanes), + n: fx.n, + frames: frames(), + }) + } + + /// The frame owner of this data. + pub fn frames(&self) -> &FrameStore { + &self.frames + } + + /// Mutable frame owner, for advancing the source after a change. + pub fn frames_mut(&mut self) -> &mut FrameStore { + &mut self.frames + } + + /// Mutable view of physical row `r` (to model a source-side change). + pub fn row_mut(&mut self, r: usize) -> &mut [u16] { + let n = self.n; + match &mut self.layout { + Layout::Slab { data, pitch } => &mut data[r * *pitch..r * *pitch + n], + Layout::Scattered(lanes) => &mut lanes[r], + } + } + + /// Physical row `r`, borrowed in place. + pub fn row(&self, r: usize) -> Option> { + match &self.layout { + Layout::Slab { data, pitch } => { + (r * pitch + self.n <= data.len()).then_some(SlabLane { + slab: data, + row: r, + pitch: *pitch, + n: self.n, + }) + } + Layout::Scattered(lanes) => lanes.get(r).map(|lane| SlabLane { + slab: lane, + row: 0, + pitch: self.n, + n: self.n, + }), + } + } + + /// Bytes this source holds. + pub fn resident_bytes(&self) -> usize { + match &self.layout { + Layout::Slab { data, .. } => data.len() * 2, + Layout::Scattered(lanes) => lanes.iter().map(|l| l.len() * 2).sum(), + } + } +} + +fn fill(fx: &Fixture, p: usize, lane: &mut [u16]) -> Result<(), IngestRefusal> { + for (i, slot) in lane.iter_mut().enumerate() { + let value = fx.value(i, p); + *slot = bf16_exact(value).ok_or(IngestRefusal::NotBf16Exact { + identity: i, + prop: p, + value, + })?; + } + Ok(()) +} + +/// The DT64 socket over a [`Bf16Lanes`]: field `f` lives in physical row +/// `rows[f]`, in row order `spaces[f]`. Counts what was borrowed. +pub struct Socket<'a> { + pub lanes: &'a Bf16Lanes, + pub rows: Vec, + pub spaces: Vec, + pub scope: Scope, + pub borrows: Cell, +} + +impl<'a> Socket<'a> { + /// Field `f` at row `rows[f]`, every lane in [`SPACE_A`]. + pub fn new(lanes: &'a Bf16Lanes, rows: Vec) -> Socket<'a> { + let spaces = vec![SPACE_A; rows.len()]; + Socket { + lanes, + rows, + spaces, + scope: READER, + borrows: Cell::new(0), + } + } + + /// Field `f` at row `f`. + pub fn in_order(lanes: &'a Bf16Lanes, props: usize) -> Socket<'a> { + Socket::new(lanes, (0..props).collect()) + } +} + +impl LaneSource for Socket<'_> { + type Lane<'b> + = SlabLane<'b> + where + Self: 'b; + + fn lane_of( + &self, + class: u32, + rev: SchemaRev, + field: u8, + _basis: &FieldRef, + _frame: &Frame, + ) -> Option { + let f = usize::from(field); + if class != CLASS || rev != SchemaRev(1) || f >= self.rows.len() { + return None; + } + Some(LaneAddr { + space: self.spaces[f], + lane: u16::try_from(self.rows[f]).ok()?, + scope: self.scope, + }) + } + + fn borrow(&self, addr: &LaneAddr) -> Option> { + self.borrows.set(self.borrows.get() + 1); + self.lanes.row(usize::from(addr.lane)) + } + + fn frames(&self) -> &FrameStore { + self.lanes.frames() + } +} diff --git a/crates/dr-isa-stream-lab/tests/integration/geometry.rs b/crates/dr-isa-stream-lab/tests/integration/geometry.rs new file mode 100644 index 000000000..9450db938 --- /dev/null +++ b/crates/dr-isa-stream-lab/tests/integration/geometry.rs @@ -0,0 +1,182 @@ +//! When A can be read in place, and what happens when it cannot. + +use dr_isa_stream_lab::bind::{World, SPACE_A}; +use dr_isa_stream_lab::cube::WhereMasks; +use dr_isa_stream_lab::fixture::{Fixture, MASKS}; +use dr_isa_stream_lab::gemm::{sums, DirectRefusal, GemmRefusal, Mode}; +use dr_isa_stream_lab::oracle; +use dr_isa_stream_lab::source::{Bf16Lanes, Socket}; + +fn want(fx: &Fixture, props: &[usize]) -> Vec<[f32; MASKS]> { + oracle::sums(fx, props) + .iter() + .map(|r| r.map(|v| v as f32)) + .collect() +} + +/// Separately allocated lanes cannot be one tile load: direct refuses, +/// staging answers correctly and reports what it copied. +#[test] +fn scattered_lanes_refuse_direct_and_stage_correctly() { + let fx = Fixture { + n: 1024, + props: 16, + seed: 11, + }; + let lanes = Bf16Lanes::scattered(&fx).unwrap(); + let socket = Socket::in_order(&lanes, 16); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + assert_eq!( + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap_err(), + GemmRefusal::Direct(DirectRefusal::NotOneSlab { field: 1 }) + ); + let (got, report) = sums(&world, &binding, &socket, &masks, &fields, Mode::Staged).unwrap(); + assert_eq!(got, want(&fx, &(0..16).collect::>())); + assert_eq!( + report.a_staged_bytes, + 16 * 1024 * 2, + "every A byte staged once" + ); +} + +/// A slab whose fields are not consecutive rows refuses direct, and is +/// correct staged. +#[test] +fn permuted_rows_refuse_direct_and_stage_correctly() { + let fx = Fixture { + n: 512, + props: 16, + seed: 12, + }; + let rows: Vec = (0..16).rev().collect(); + let lanes = Bf16Lanes::slab(&fx, &rows).unwrap(); + let socket = Socket::new(&lanes, rows); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + assert_eq!( + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap_err(), + GemmRefusal::Direct(DirectRefusal::RowsNotConsecutive { field: 1 }) + ); + let (got, _) = sums(&world, &binding, &socket, &masks, &fields, Mode::Staged).unwrap(); + assert_eq!(got, want(&fx, &(0..16).collect::>())); +} + +/// The row check is load-bearing: the same permuted slab read as if its rows +/// were in field order gives a different answer from the oracle. +#[test] +fn without_the_row_check_a_permuted_slab_reads_wrong() { + let fx = Fixture { + n: 512, + props: 16, + seed: 12, + }; + let rows: Vec = (0..16).rev().collect(); + let lanes = Bf16Lanes::slab(&fx, &rows).unwrap(); + // A socket that claims field f is row f: what an unchecked direct read + // of "16 rows from the first lane's row" would assume. + let lying = Socket::in_order(&lanes, 16); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&lying, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let (got, _) = sums(&world, &binding, &lying, &masks, &fields, Mode::Direct).unwrap(); + assert_ne!(got, want(&fx, &(0..16).collect::>())); +} + +/// Fewer than 16 demanded lanes is a partial row tile: direct refuses (it +/// would read undemanded lanes), staging answers with zero rows. +#[test] +fn partial_row_tiles_are_staged() { + for props in [3usize, 8] { + let fx = Fixture { + n: 700, + props, + seed: 13, + }; + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let socket = Socket::in_order(&lanes, props); + let world = World::new(); + let fields: Vec = (0..props as u8).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + assert_eq!( + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap_err(), + GemmRefusal::Direct(DirectRefusal::PartialRowTile { lanes: props }) + ); + let (got, _) = sums(&world, &binding, &socket, &masks, &fields, Mode::Staged).unwrap(); + assert_eq!( + got, + want(&fx, &(0..props).collect::>()), + "props={props}" + ); + } +} + +/// Reusing the selection: 32 lanes are two row tiles over one staged B per +/// block, so B is staged once, not once per tile. +#[test] +fn b_is_staged_once_per_block_for_all_row_tiles() { + let fx = Fixture { + n: 2048, + props: 32, + seed: 14, + }; + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let socket = Socket::in_order(&lanes, 32); + let world = World::new(); + let fields32: Vec = (0..32).collect(); + let binding = world.bind(&socket, &fields32, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let (_, r32) = sums(&world, &binding, &socket, &masks, &fields32, Mode::Direct).unwrap(); + let (_, r16) = sums( + &world, + &binding, + &socket, + &masks, + &fields32[..16], + Mode::Direct, + ) + .unwrap(); + assert_eq!(r32.b_entries, r16.b_entries); + assert_eq!(r32.gemms, 2 * r16.gemms); + assert_eq!(r32.b_entries as u32, masks.counts().iter().sum::()); +} + +/// A pitch that leaves no padding for the last 32-wide block refuses direct +/// (the read would run into the next row or past the slab), and stages. +#[test] +fn an_unpadded_pitch_refuses_direct_and_stages_correctly() { + let fx = Fixture { + n: 513, + props: 16, + seed: 15, + }; + let rows: Vec = (0..16).collect(); + let lanes = Bf16Lanes::slab_with_pitch(&fx, &rows, 513).unwrap(); + let socket = Socket::new(&lanes, rows); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + assert_eq!( + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap_err(), + GemmRefusal::Direct(DirectRefusal::ShortPitch { + pitch: 513, + needed: 544 + }) + ); + let (got, _) = sums(&world, &binding, &socket, &masks, &fields, Mode::Staged).unwrap(); + assert_eq!(got, want(&fx, &(0..16).collect::>())); + // A padded pitch over the same values is read in place. + let padded = Bf16Lanes::slab_with_pitch(&fx, &(0..16).collect::>(), 544).unwrap(); + let socket = Socket::in_order(&padded, 16); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let (got, r) = sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap(); + assert_eq!(got, want(&fx, &(0..16).collect::>())); + assert_eq!(r.a_staged_bytes, 0); +} diff --git a/crates/dr-isa-stream-lab/tests/integration/main.rs b/crates/dr-isa-stream-lab/tests/integration/main.rs new file mode 100644 index 000000000..91492ceb1 --- /dev/null +++ b/crates/dr-isa-stream-lab/tests/integration/main.rs @@ -0,0 +1,4 @@ +pub mod geometry; +pub mod numeric; +pub mod parity; +pub mod refusals; diff --git a/crates/dr-isa-stream-lab/tests/integration/numeric.rs b/crates/dr-isa-stream-lab/tests/integration/numeric.rs new file mode 100644 index 000000000..64c7eaf0e --- /dev/null +++ b/crates/dr-isa-stream-lab/tests/integration/numeric.rs @@ -0,0 +1,28 @@ +//! bf16 is a numeric realization, not the value's meaning: an exactness +//! claim is refused where bf16 cannot hold the value. + +use dr_isa_stream_lab::fixture::{bf16_exact, bf16_round, bf16_value}; + +#[test] +fn small_integers_are_exact_and_257_is_not() { + for v in -256..=256 { + assert_eq!(bf16_value(bf16_exact(v).unwrap()) as i32, v, "{v}"); + } + assert!(bf16_exact(257).is_none(), "bf16 has 8 significant bits"); + assert!(bf16_exact(1_000_001).is_none()); + assert_eq!(bf16_exact(1 << 20).map(bf16_value), Some((1 << 20) as f32)); +} + +/// A decimal-scaled value (cents as a float) is approximate in bf16; the +/// error is measured, not hidden. +#[test] +fn approximate_values_report_their_error() { + let cents = [1.99f32, 12.34, 99.99, 1234.56]; + let mut worst = 0f32; + for &v in ¢s { + let back = bf16_value(bf16_round(v)); + worst = worst.max(((back - v) / v).abs()); + } + assert!(worst > 0.0, "these values are not exact in bf16"); + assert!(worst <= 2f32.powi(-8), "round-to-nearest bound: {worst}"); +} diff --git a/crates/dr-isa-stream-lab/tests/integration/parity.rs b/crates/dr-isa-stream-lab/tests/integration/parity.rs new file mode 100644 index 000000000..2f8d65103 --- /dev/null +++ b/crates/dr-isa-stream-lab/tests/integration/parity.rs @@ -0,0 +1,135 @@ +//! Both arms against the fixture oracle, over one binding. + +use dr_isa_stream_lab::bind::{World, SPACE_A}; +use dr_isa_stream_lab::cube::{fold, WhereMasks}; +use dr_isa_stream_lab::fixture::{Fixture, MASKS}; +use dr_isa_stream_lab::gemm::{sums, sums_whole, Mode, SelectionB}; +use dr_isa_stream_lab::oracle; +use dr_isa_stream_lab::source::{Bf16Lanes, Socket}; + +fn as_f32(rows: &[[i64; MASKS]]) -> Vec<[f32; MASKS]> { + rows.iter().map(|r| r.map(|v| v as f32)).collect() +} + +/// One binding, both arms, the oracle: exact for every listed size. +#[test] +fn one_binding_serves_the_cube_fold_and_the_tile_gemm_exactly() { + for (n, props) in [(256, 16), (513, 16), (4096, 32), (65_536, 16)] { + let fx = Fixture { + n, + props, + seed: 0x15A_0000 + n as u64, + }; + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let socket = Socket::in_order(&lanes, props); + let world = World::new(); + let fields: Vec = (0..props as u8).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let props_us: Vec = (0..props).collect(); + let want = oracle::sums(&fx, &props_us); + + let cube = fold(&world, &binding, &socket, &masks, &fields).unwrap(); + assert_eq!(cube, want, "cube fold, n={n}"); + + let (direct, report) = + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap(); + assert_eq!( + direct, + as_f32(&want), + "direct GEMM, n={n}, tier {}", + report.tier + ); + assert_eq!(report.a_staged_bytes, 0, "direct read copied A, n={n}"); + assert_eq!(report.direct_tiles, props / 16); + + let (staged, report) = + sums(&world, &binding, &socket, &masks, &fields, Mode::Staged).unwrap(); + assert_eq!(staged, as_f32(&want), "staged GEMM, n={n}"); + assert!(report.a_staged_bytes > 0); + + let sel = SelectionB::stage(&masks); + let (whole, report) = + sums_whole(&world, &binding, &socket, &sel, &fields, Mode::Direct).unwrap(); + assert_eq!(whole, as_f32(&want), "whole-lane direct GEMM, n={n}"); + assert_eq!((report.a_staged_bytes, report.gemms), (0, props / 16)); + let (whole, report) = + sums_whole(&world, &binding, &socket, &sel, &fields, Mode::Staged).unwrap(); + assert_eq!(whole, as_f32(&want), "whole-lane staged GEMM, n={n}"); + assert!(report.a_staged_bytes > 0); + } +} + +/// The fixture is not degenerate: masks differ in density and the sums are +/// not all equal, so a wrong mask or lane cannot pass by coincidence. +#[test] +fn the_fixture_discriminates() { + let fx = Fixture { + n: 4096, + props: 16, + seed: 7, + }; + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let counts = masks.counts(); + assert!( + counts.windows(2).all(|w| w[0] < w[1]), + "mask densities {counts:?}" + ); + assert!(counts[0] > 0 && (counts[MASKS - 1] as usize) < fx.n); + let want = oracle::sums(&fx, &(0..16).collect::>()); + let distinct: std::collections::BTreeSet = want.iter().flatten().copied().collect(); + assert!( + distinct.len() > 100, + "only {} distinct sums", + distinct.len() + ); +} + +/// The binding resolves each demanded lane once per arm, independent of n: +/// no per-identity lookup happens on the hot path. +#[test] +fn lanes_are_resolved_once_per_arm_not_per_identity() { + for n in [256, 65_536] { + let fx = Fixture { + n, + props: 16, + seed: 3, + }; + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let socket = Socket::in_order(&lanes, 16); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let before = socket.borrows.get(); + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap(); + assert_eq!(socket.borrows.get() - before, 16, "GEMM borrows, n={n}"); + let before = socket.borrows.get(); + fold(&world, &binding, &socket, &masks, &fields).unwrap(); + assert_eq!(socket.borrows.get() - before, 16, "fold borrows, n={n}"); + } +} + +/// Which GEMM tier ran is reported, never assumed. +#[test] +fn the_tier_is_reported() { + let fx = Fixture { + n: 256, + props: 16, + seed: 1, + }; + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let socket = Socket::in_order(&lanes, 16); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let (_, report) = sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap(); + eprintln!("GEMM tier: {}", report.tier); + if !report.tier.starts_with("AMX") { + eprintln!( + "SKIP (honestly): AMX not available; the direct path ran on {}", + report.tier + ); + } +} diff --git a/crates/dr-isa-stream-lab/tests/integration/refusals.rs b/crates/dr-isa-stream-lab/tests/integration/refusals.rs new file mode 100644 index 000000000..b23c751a2 --- /dev/null +++ b/crates/dr-isa-stream-lab/tests/integration/refusals.rs @@ -0,0 +1,129 @@ +//! Coordinate space, frame and mask-space refusals, and why they matter. + +use dr_isa_stream_lab::bind::{World, SPACE_A, SPACE_OTHER}; +use dr_isa_stream_lab::cube::{fold, ArmRefusal, WhereMasks}; +use dr_isa_stream_lab::fixture::{bf16_exact, Fixture, MASKS}; +use dr_isa_stream_lab::gemm::{sums, sums_whole, GemmRefusal, Mode, SelectionB}; +use dr_isa_stream_lab::oracle; +use dr_isa_stream_lab::source::{Bf16Lanes, Socket}; +use lance_graph_dt64::dr::{FrameRefusal, FrameRevisions}; +use lance_graph_dt64::LaneRefusal; + +fn fx() -> Fixture { + Fixture { + n: 1024, + props: 16, + seed: 21, + } +} + +/// A lane in another row order is refused at bind, before any read. +#[test] +fn a_lane_in_another_row_order_is_refused_at_bind() { + let fx = fx(); + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let mut socket = Socket::in_order(&lanes, 16); + socket.spaces[5] = SPACE_OTHER; + let world = World::new(); + let fields: Vec = (0..16).collect(); + match world.bind(&socket, &fields, SPACE_A) { + Err(LaneRefusal::CoordinateSpace { field: 5, .. }) => {} + other => panic!("expected a coordinate-space refusal, got {other:?}"), + } +} + +/// The refusal is load-bearing: the reordered lane, read positionally, gives +/// a wrong answer that the oracle catches. +#[test] +fn without_the_refusal_a_reordered_lane_reads_wrong() { + let fx = fx(); + let mut lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + lanes.row_mut(5).reverse(); // field 5 now in reverse row order + let socket = Socket::in_order(&lanes, 16); // ...but declared in SPACE_A + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let want: Vec<[f32; MASKS]> = oracle::sums(&fx, &(0..16).collect::>()) + .iter() + .map(|r| r.map(|v| v as f32)) + .collect(); + let (got, _) = sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap(); + assert_ne!(got[5], want[5], "the reversed lane must read wrong"); + assert_eq!(got[4], want[4], "neighbouring lanes are unaffected"); +} + +/// A source that changed after binding refuses both arms before they read. +#[test] +fn a_stale_frame_refuses_both_arms() { + let fx = fx(); + let mut lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = { + let socket = Socket::in_order(&lanes, 16); + world.bind(&socket, &fields, SPACE_A).unwrap() + }; + lanes.frames_mut().advance(FrameRevisions { + source: 2, + relation_map: 1, + coordinates: 1, + }); + let socket = Socket::in_order(&lanes, 16); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + let stale = LaneRefusal::Frame(FrameRefusal::SourceChanged); + assert_eq!( + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap_err(), + GemmRefusal::Arm(ArmRefusal::Lane(stale.clone())) + ); + assert_eq!( + fold(&world, &binding, &socket, &masks, &fields).unwrap_err(), + ArmRefusal::Lane(stale) + ); + // A fresh binding at the new frame is accepted again. + let rebound = world.bind(&socket, &fields, SPACE_A).unwrap(); + assert!(sums(&world, &rebound, &socket, &masks, &fields, Mode::Direct).is_ok()); +} + +/// Masks computed in another row order are refused by both arms. +#[test] +fn masks_in_another_row_order_are_refused() { + let fx = fx(); + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let socket = Socket::in_order(&lanes, 16); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_OTHER); + let refusal = ArmRefusal::MaskSpace { + masks: SPACE_OTHER, + selection: SPACE_A, + }; + assert_eq!( + fold(&world, &binding, &socket, &masks, &fields).unwrap_err(), + refusal + ); + assert_eq!( + sums(&world, &binding, &socket, &masks, &fields, Mode::Direct).unwrap_err(), + GemmRefusal::Arm(refusal.clone()) + ); + let sel = SelectionB::stage(&masks); + assert_eq!( + sums_whole(&world, &binding, &socket, &sel, &fields, Mode::Direct).unwrap_err(), + GemmRefusal::Arm(refusal) + ); +} + +/// Sanity of the refusal pair: equal spaces are accepted. +#[test] +fn matching_spaces_are_accepted() { + let fx = fx(); + let lanes = Bf16Lanes::slab_in_order(&fx).unwrap(); + let socket = Socket::in_order(&lanes, 16); + let world = World::new(); + let fields: Vec = (0..16).collect(); + let binding = world.bind(&socket, &fields, SPACE_A).unwrap(); + let masks = WhereMasks::from_fixture(&fx, SPACE_A); + assert!(fold(&world, &binding, &socket, &masks, &fields).is_ok()); + assert!(bf16_exact(1).is_some()); +} From d15115a9d54f8fe17b8499302e08e79b2b989f1c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 8 Nov 2026 00:25:55 +0000 Subject: [PATCH 2/3] Add the lab's CI workflow and board entry The workflow runs fmt, clippy, the tests and the stage-cost probe on the runner's own tier and prints which tier ran. The entry records the contract map, the result matrix, the measured costs and the handoff for report-rs #33. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- .../entries/2026-11-08-dr-isa-stream-lab.md | 126 ++++++++++++++++++ .claude/board/entries/README.md | 3 +- .github/workflows/dr-isa-stream-lab.yml | 73 ++++++++++ 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-11-08-dr-isa-stream-lab.md create mode 100644 .github/workflows/dr-isa-stream-lab.yml diff --git a/.claude/board/entries/2026-11-08-dr-isa-stream-lab.md b/.claude/board/entries/2026-11-08-dr-isa-stream-lab.md new file mode 100644 index 000000000..a6fe45a8f --- /dev/null +++ b/.claude/board/entries/2026-11-08-dr-isa-stream-lab.md @@ -0,0 +1,126 @@ +# 2026-11-08 — DR G2L × ISA stream lab: one lane binding, two computation geometries + +**Status:** MEASURED (lab, not production admission). Crate: +`crates/dr-isa-stream-lab`. Depends on AdaWorldAPI/ndarray#414 +(`bf16_tile_gemm_16x16_packed_strided`). + +**Question.** Can a source-witnessed DT64 lane binding be consumed by both an +ordered-cube fold and an AMX BF16 GEMM without a new canonical value layout, +and what is the minimum physical staging? + +**Answer.** Yes. Source data is read in place (direct stream). The selection +needs one bounded staging step. + +- A (the source's lanes) is loaded straight into AMX tiles from the slab when + the demanded lanes sit in one allocation, at consecutive rows, with a + uniform pitch padded to 32. +- B is the WHERE result itself, a 0/1 matrix, staged once per mask set. +- No DR or DT64 change was needed. +- One ndarray API gap blocked the direct read: the safe GEMM entry hardcoded + A's row stride to K. ndarray#414 closes it. + +## Shape + +`C[p][m] = Σᵢ lane_p[i] · mask_m(i)`, i.e. every `SUM(lane p) WHERE mask m` +in one GEMM: M = 16 demanded lanes, N = 16 WHERE masks, K = the padded +population. ARM A computes the same 16×16 answer as a positional fold over +the 256×256 cube (`tile(i) = i >> 8`, `slot(i) = i & 255`). Both arms take one +`LaneBinding` and match an oracle computed from the fixture alone. + +## Contract map + +1. OGAR / ClassView. A 32-field view (`bind.rs`). +2. DT64. `BindStore::register` / `route` / `bind_lanes` give a `LaneBinding` + (`lance-graph-dt64/src/lane.rs:28-140`, `:352`). It holds addresses only: + `LaneAddr { space: CoordSpace, lane, scope }`. +3. Source. `LaneSource::borrow` returns the source's own `Lane<'a>`. Here that + is `SlabLane { slab, row, pitch, n }`, so **the physical geometry rides on + the source's lane type, not on DR**. `FrameStore` is owned by the data. +4. Consumer. ARM B checks in safe code that the 16 borrowed lanes share one + allocation (`ptr::eq`), have equal pitch, sit at consecutive rows, and have + a pitch of at least `round32(n)`. It then calls the strided GEMM with + stride = pitch. +5. ndarray. `TILELOADD` takes the stride as an operand (`amx_matmul.rs` + `tile_load`). Before #414, `amx_path` passed `k*2`. + +**First missing guarantee.** The trait cannot prove that a source's +`SlabLane` geometry describes its real memory. That is the same obligation +`LaneSource::frames` already documents, and it stays the implementer's. + +## Result matrix + +| Claim | State | +|---|---| +| Both arms exact vs oracle at n = 256, 513, 4096, 65536 | proven in lab | +| One binding, 16 lane resolutions per arm, independent of n | proven in lab | +| Direct A (in place) on AMX, zero A bytes copied | proven in lab (AMX TDPBF16PS) | +| Lane in another CoordSpace refused at bind | existing DT64, exercised | +| …and read positionally it gives a wrong answer | proven (oracle catches it) | +| Stale frame refuses both arms; re-bind accepted | existing DT64, exercised | +| Masks / selection in another space refused | lab guard | +| Scattered lanes, permuted rows, unpadded pitch, < 16 lanes: direct refused, staged correct | lab guards | +| bf16 exactness: ints in [-256, 256] exact, 257 refused; decimals approximate, error ≤ 2⁻⁸ | lab | +| MKL comparator | **not executed** (MKL not installed) | +| Non-AMX tier | CI runner (tier printed in the log) | + +Disable runs: removing each of the six lab guards fails exactly the test +that names it. + +## Costs + +Granite Rapids VM, 4 vCPU, AMX TDPBF16PS, rustc 1.99.0, lance-graph `7c301f5`, +ndarray `9a632f3`. Warm medians of 41 runs. Every timed result is checked +against the oracle. + +n = 65,536, 16 lanes, 16 masks (524,717 selected bits): + +| Stage | Cost | +|---|---| +| bind (cold, once) | 1.0 µs | +| ARM A: 16 WHERE counts (popcount terminal) | 2.2 µs | +| stage selection B (cold, once per mask set; 2 MiB) | 689 µs | +| ARM B whole lane, A in place | **152 µs** | +| ARM B whole lane, A staged (copy + zero-fill) | 362 µs | +| reference: per-32-block GEMM calls, B restaged per block | 2,212 µs | +| reference: ARM A per-bit sum fold (scalar) | 13,130 µs | + +At 32 lanes, B is reused across both row tiles: 311 µs direct versus 726 µs +staged. + +**Per-call overhead, measured.** AMX costs about 43 ns per 16×16×32 block in +steady state, plus about 105 ns fixed per call (tile config and release). A +GEMM of K = 32 per call therefore never amortizes. A consumer must make one +call per row tile with K = the population. + +The two reference rows are not gates. They show where the cost goes when the +work leaves the fold: per-block calls, and a scalar per-bit gather for a sum +that is GEMM-shaped. + +## Smallest justified interface change + +- ndarray: `bf16_tile_gemm_16x16_packed_strided` (#414). +- DR and DT64: none. + +A source that wants direct AMX consumption exposes a lane type carrying +`(allocation, row, pitch)`. DR keeps describing identity and row order only. + +## Open + +- B staging costs 689 µs cold. A sequential bit-to-VNNI expansion should be + faster; not attempted. +- Gotcha 14: AMX tile state can corrupt on an oversubscribed VM. The numbers + above were verified on every run, but this is not certification on + dedicated silicon. +- MKL comparison not run. +- 32×128 and 64×64 layouts not compared. + +## Handoff for report-rs #33 + +- **Available as lab primitives:** + - the strided BF16 GEMM (once #414 merges); + - the `SlabLane` geometry pattern; + - a SUM-WHERE-for-16-masks GEMM recipe. +- **Lab-only:** everything in `dr-isa-stream-lab`; nothing is a production + path. +- **Unchanged for #33:** its workbook gates. bf16 is a numeric realization; + an exact decimal must not be treated as bf16. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index ed544d5f1..180b28810 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -586 entries, 2026-08-06 .. 2026-11-07. +587 entries, 2026-08-06 .. 2026-11-08. | date | entry id | finding | file | |---|---|---|---| +| 2026-11-08 | `dr-isa-stream-lab` | | [2026-11-08-dr-isa-stream-lab.md](2026-11-08-dr-isa-stream-lab.md) | | 2026-11-07 | `D-DT64-7-L5b` | | [2026-11-07-dt64-l5b-historical-basis.md](2026-11-07-dt64-l5b-historical-basis.md) | | 2026-11-07 | `D-DT64-7-L5a` | | [2026-11-07-dt64-l5a-exact-basis-witness.md](2026-11-07-dt64-l5a-exact-basis-witness.md) | | 2026-11-07 | `D-DT64-7-L5` | | [2026-11-07-dt64-decision-j-exact-basis.md](2026-11-07-dt64-decision-j-exact-basis.md) | diff --git a/.github/workflows/dr-isa-stream-lab.yml b/.github/workflows/dr-isa-stream-lab.yml new file mode 100644 index 000000000..0f50b837c --- /dev/null +++ b/.github/workflows/dr-isa-stream-lab.yml @@ -0,0 +1,73 @@ +name: DR ISA Stream Lab +# Own [workspace] excluded from root tests; a lab gate, not production +# admission. GitHub runners have no AMX: the tests still prove the stream +# contract and the refusals on whatever tier the runner has, and the run log +# names that tier. AMX numbers come from a host that has it. +on: + pull_request: + paths: + - "crates/dr-isa-stream-lab/**" + - "crates/lance-graph-dt64/**" + - "crates/lance-graph-contract/**" + - ".github/workflows/dr-isa-stream-lab.yml" + - "rust-toolchain.toml" + push: + branches: [main] + paths: + - "crates/dr-isa-stream-lab/**" + - "crates/lance-graph-dt64/**" + - "crates/lance-graph-contract/**" + - ".github/workflows/dr-isa-stream-lab.yml" + - "rust-toolchain.toml" + workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + RUST_BACKTRACE: "1" +jobs: + isolated-probe: + name: excluded-crate fmt, clippy and tests + runs-on: ubuntu-24.04 + timeout-minutes: 60 + defaults: + run: + working-directory: lance-graph + steps: + - uses: actions/checkout@v4 + with: + path: lance-graph + persist-credentials: false + - name: Checkout AdaWorldAPI/ndarray (sibling dependency) + uses: actions/checkout@v4 + with: + repository: AdaWorldAPI/ndarray + path: ndarray + persist-credentials: false + - name: Rust toolchain + run: rustup show + - name: Resolve target-cpu=native for this runner + id: cpu + run: | + flags=$(.github/scripts/resolve-native-rustflags.sh) + echo "RUSTFLAGS=$flags" >> "$GITHUB_ENV" + hash=$(printf %s "$flags" | sha256sum | cut -c1-16) + echo "flags=$hash" >> "$GITHUB_OUTPUT" + - uses: Swatinem/rust-cache@v2 + with: + key: cpu-${{ steps.cpu.outputs.flags }} + workspaces: lance-graph/crates/dr-isa-stream-lab + - name: fmt + run: cargo fmt --manifest-path crates/dr-isa-stream-lab/Cargo.toml --check + - name: clippy + run: cargo clippy --manifest-path crates/dr-isa-stream-lab/Cargo.toml --all-targets -- -D warnings + - name: tests (prints the GEMM tier) + run: cargo test --release --manifest-path crates/dr-isa-stream-lab/Cargo.toml -- --nocapture + - name: stage costs (every timed result is checked against the oracle) + run: cargo run --release --manifest-path crates/dr-isa-stream-lab/Cargo.toml --bin bench From b7922b89af15c854ba29bfd025bb5472e7cc8fc3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 8 Nov 2026 01:01:00 +0000 Subject: [PATCH 3/3] Compare bf16 exactness in f64, not through a saturating cast `i32::MAX as f32` rounds to 2^31, whose low 16 bits are zero, and casting it back to i32 saturates to i32::MAX, so the check passed a value bf16 cannot hold. f64 holds every i32 and every f32 exactly. The boundary test covers i32::MAX, i32::MAX - 1 and i32::MIN + 1 (refused) and i32::MIN, +-2^30 and 3*2^28 (exact, decoded in f64). It failed on i32::MAX before the fix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMWgnUU9g7Ldyn6sbk --- crates/dr-isa-stream-lab/src/fixture.rs | 4 +++- .../dr-isa-stream-lab/tests/integration/numeric.rs | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/dr-isa-stream-lab/src/fixture.rs b/crates/dr-isa-stream-lab/src/fixture.rs index c7f51ad60..7b115bf11 100644 --- a/crates/dr-isa-stream-lab/src/fixture.rs +++ b/crates/dr-isa-stream-lab/src/fixture.rs @@ -42,7 +42,9 @@ impl Fixture { pub fn bf16_exact(v: i32) -> Option { let f = v as f32; let bits = f.to_bits(); - if bits & 0xFFFF != 0 || f as i32 != v { + // Compare in f64, which holds every i32 and every f32 exactly: casting + // `f` back to i32 would saturate 2^31 to i32::MAX and pass it as exact. + if bits & 0xFFFF != 0 || f64::from(f) != f64::from(v) { None } else { Some((bits >> 16) as u16) diff --git a/crates/dr-isa-stream-lab/tests/integration/numeric.rs b/crates/dr-isa-stream-lab/tests/integration/numeric.rs index 64c7eaf0e..fa4e91178 100644 --- a/crates/dr-isa-stream-lab/tests/integration/numeric.rs +++ b/crates/dr-isa-stream-lab/tests/integration/numeric.rs @@ -13,6 +13,20 @@ fn small_integers_are_exact_and_257_is_not() { assert_eq!(bf16_exact(1 << 20).map(bf16_value), Some((1 << 20) as f32)); } +/// At the edge of `i32`, `v as f32` rounds up to 2^31, which a cast back to +/// `i32` saturates to `i32::MAX`. The check must not be fooled by that +/// saturation. Decoded values are compared in f64, which holds both exactly. +#[test] +fn exactness_is_not_fooled_by_saturating_casts() { + for v in [i32::MAX, i32::MAX - 1, i32::MIN + 1] { + assert!(bf16_exact(v).is_none(), "{v} is not a bf16 value"); + } + for v in [i32::MIN, 1 << 30, -(1 << 30), 3 << 28] { + let b = bf16_exact(v).unwrap_or_else(|| panic!("{v} is a bf16 value")); + assert_eq!(f64::from(bf16_value(b)), f64::from(v), "{v}"); + } +} + /// A decimal-scaled value (cents as a float) is approximate in bf16; the /// error is measured, not hidden. #[test]