From 05a91b0a04ea9bd91151a901ac3cdc8f3570e7cf Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 09:05:52 +0000 Subject: [PATCH 1/2] Complete the U16x32 facade surface on every backend U16x32 had a different method set on each backend, so code written against ndarray::simd could not use zero(), the u16->f32 widening, the compare masks, min/max or saturating arithmetic portably. Every backend (AVX-512, AVX2, AVX, scalar, nightly) now carries the same surface: zero, simd_min/max, saturating_add/sub, cmpeq_mask/cmpgt_mask, reduce_min/max, to_f32x16_lo/hi, the bit operators, Not, SubAssign and Default, and the widening, shift, multiply and pack methods on nightly. reduce_sum returns u16 everywhere. The new parity test also caught two AVX-512 bugs: shl/shr returned 0 for any count other than 1, 2, 4 or 8 (including 0), and pack_saturate_u8 saturated as signed and interleaved its inputs per 128-bit lane. Both now match the scalar reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMjKnig6MXTd41Lf2v --- src/simd.rs | 119 +++++++++++++++ src/simd_avx.rs | 154 +++++++++++++++++++- src/simd_avx2.rs | 233 +++++++++++++++++++++++++++++ src/simd_avx512.rs | 242 +++++++++++++++++++++++++++---- src/simd_nightly/u_word_types.rs | 69 +++++++++ src/simd_scalar.rs | 130 +++++++++++++++++ 6 files changed, 921 insertions(+), 26 deletions(-) diff --git a/src/simd.rs b/src/simd.rs index 5e9f0ab5..9b9c7023 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -2194,3 +2194,122 @@ mod tests { } } } + +/// Parity of the `U16x32` surface against a plain-array reference, run on +/// whichever backend this build selected (AVX-512, AVX2, AVX, scalar, NEON, +/// wasm or nightly). The corpus mixes a fixed-seed generator with the edge +/// values a u16 lane can hit: 0, 1, 255, 256, 0x7FFF, 0x8000, 0xFFFF. +#[cfg(test)] +mod u16x32_facade_parity { + use super::{F32x16, U16x32, U8x64}; + + const EDGES: [u16; 7] = [0, 1, 255, 256, 0x7FFF, 0x8000, 0xFFFF]; + + fn corpus() -> Vec<[u16; 32]> { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + (state >> 48) as u16 + }; + let mut out: Vec<[u16; 32]> = (0..64).map(|_| core::array::from_fn(|_| next())).collect(); + for &e in &EDGES { + out.push([e; 32]); + } + out.push(core::array::from_fn(|i| EDGES[i % EDGES.len()])); + out.push(core::array::from_fn(|i| EDGES[(i * 3 + 1) % EDGES.len()])); + out + } + + fn v(a: [u16; 32]) -> U16x32 { + U16x32::from_array(a) + } + + #[test] + fn elementwise_ops_match_reference() { + let c = corpus(); + let mut checked = 0; + for a in &c { + for b in &c { + let (x, y) = (v(*a), v(*b)); + let r = |f: fn(u16, u16) -> u16| -> [u16; 32] { core::array::from_fn(|i| f(a[i], b[i])) }; + assert_eq!(x.simd_min(y).to_array(), r(u16::min), "simd_min {a:?} {b:?}"); + assert_eq!(x.simd_max(y).to_array(), r(u16::max), "simd_max"); + assert_eq!(x.saturating_add(y).to_array(), r(u16::saturating_add), "saturating_add"); + assert_eq!(x.saturating_sub(y).to_array(), r(u16::saturating_sub), "saturating_sub"); + assert_eq!(x.mullo(y).to_array(), r(u16::wrapping_mul), "mullo"); + assert_eq!((x + y).to_array(), r(u16::wrapping_add), "add"); + assert_eq!((x - y).to_array(), r(u16::wrapping_sub), "sub"); + let eq = (0..32).fold(0u32, |m, i| m | (((a[i] == b[i]) as u32) << i)); + let gt = (0..32).fold(0u32, |m, i| m | (((a[i] > b[i]) as u32) << i)); + assert_eq!(x.cmpeq_mask(y), eq, "cmpeq_mask {a:?} {b:?}"); + assert_eq!(x.cmpgt_mask(y), gt, "cmpgt_mask {a:?} {b:?}"); + checked += 1; + } + } + assert!(checked > 5000, "corpus too small: {checked}"); + } + + #[test] + fn reductions_and_conversions_match_reference() { + for a in corpus() { + let x = v(a); + assert_eq!(x.reduce_min(), *a.iter().min().unwrap(), "reduce_min {a:?}"); + assert_eq!(x.reduce_max(), *a.iter().max().unwrap(), "reduce_max {a:?}"); + assert_eq!(x.reduce_sum(), a.iter().fold(0u16, |s, &l| s.wrapping_add(l)), "reduce_sum {a:?}"); + let lo: [f32; 16] = core::array::from_fn(|i| a[i] as f32); + let hi: [f32; 16] = core::array::from_fn(|i| a[16 + i] as f32); + assert_eq!(x.to_f32x16_lo().to_array(), lo, "to_f32x16_lo {a:?}"); + assert_eq!(x.to_f32x16_hi().to_array(), hi, "to_f32x16_hi {a:?}"); + for s in [0u32, 1, 7, 8, 15, 16, 31] { + let shl: [u16; 32] = core::array::from_fn(|i| if s < 16 { a[i] << s } else { 0 }); + let shr: [u16; 32] = core::array::from_fn(|i| if s < 16 { a[i] >> s } else { 0 }); + assert_eq!(x.shl(s).to_array(), shl, "shl {s}"); + assert_eq!(x.shr(s).to_array(), shr, "shr {s}"); + } + } + assert_eq!(U16x32::zero().to_array(), [0u16; 32]); + let _: F32x16 = U16x32::zero().to_f32x16_lo(); + } + + #[test] + fn widening_and_narrowing_match_reference() { + let bytes: [u8; 64] = core::array::from_fn(|i| (i as u8).wrapping_mul(37).wrapping_add(200)); + let b = U8x64::from_array(bytes); + let lo: [u16; 32] = core::array::from_fn(|i| bytes[i] as u16); + let hi: [u16; 32] = core::array::from_fn(|i| bytes[32 + i] as u16); + assert_eq!(U16x32::from_u8x64_lo(b).to_array(), lo); + assert_eq!(U16x32::from_u8x64_hi(b).to_array(), hi); + for a in corpus() { + let other: [u16; 32] = core::array::from_fn(|i| a[31 - i]); + let want: [u8; 64] = core::array::from_fn(|i| { + if i < 32 { + a[i].min(255) as u8 + } else { + other[i - 32].min(255) as u8 + } + }); + assert_eq!(v(a).pack_saturate_u8(v(other)).to_array(), want, "pack_saturate_u8 {a:?}"); + } + } + + #[test] + fn bitwise_ops_match_reference() { + let c = corpus(); + for a in &c { + for b in c.iter().take(8) { + let (x, y) = (v(*a), v(*b)); + let r = |f: fn(u16, u16) -> u16| -> [u16; 32] { core::array::from_fn(|i| f(a[i], b[i])) }; + assert_eq!((x & y).to_array(), r(|p, q| p & q)); + assert_eq!((x | y).to_array(), r(|p, q| p | q)); + assert_eq!((x ^ y).to_array(), r(|p, q| p ^ q)); + let mut z = x; + z -= y; + assert_eq!(z.to_array(), r(u16::wrapping_sub)); + } + assert_eq!((!v(*a)).to_array(), a.map(|l| !l)); + } + assert_eq!(U16x32::default().to_array(), [0u16; 32]); + } +} diff --git a/src/simd_avx.rs b/src/simd_avx.rs index 084aef02..a9013156 100644 --- a/src/simd_avx.rs +++ b/src/simd_avx.rs @@ -28,7 +28,7 @@ use core::ops::{ Add, AddAssign, BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Mul, Not, Shl, Shr, Sub, SubAssign, }; -use crate::simd_avx2::{I32x16, U16x16, U64x8, U8x32}; +use crate::simd_avx2::{F32x16, I32x16, U16x16, U16x32, U64x8, U8x32}; use crate::simd_avx512::{F32x8, I16x16, I8x32}; // --------------------------------------------------------------------------- @@ -815,3 +815,155 @@ impl SubAssign for I16x16 { *self = I16x16::sub(*self, rhs); } } + +// --------------------------------------------------------------------------- +// U16x32 (64-byte array) — the AVX2-gated methods, as four 128-bit quarters +// --------------------------------------------------------------------------- + +impl U16x32 { + /// The four 128-bit quarters of the 64-byte array (lanes 0..8, 8..16, + /// 16..24, 24..32). + #[inline(always)] + fn quarters(self) -> [__m128i; 4] { + // SAFETY: SSE2 is compiled in (module docs). The array is 64 bytes, so + // four 16-byte unaligned loads are in bounds. + unsafe { + let p = self.0.as_ptr() as *const __m128i; + [_mm_loadu_si128(p), _mm_loadu_si128(p.add(1)), _mm_loadu_si128(p.add(2)), _mm_loadu_si128(p.add(3))] + } + } + + #[inline(always)] + fn from_quarters(q: [__m128i; 4]) -> Self { + let mut o = [0u16; 32]; + // SAFETY: four 16-byte stores into a 64-byte array. + unsafe { + let p = o.as_mut_ptr() as *mut __m128i; + _mm_storeu_si128(p, q[0]); + _mm_storeu_si128(p.add(1), q[1]); + _mm_storeu_si128(p.add(2), q[2]); + _mm_storeu_si128(p.add(3), q[3]); + } + Self(o) + } + + #[inline(always)] + fn zip(self, other: Self, f: impl Fn(__m128i, __m128i) -> __m128i) -> Self { + let a = self.quarters(); + let b = other.quarters(); + Self::from_quarters([f(a[0], b[0]), f(a[1], b[1]), f(a[2], b[2]), f(a[3], b[3])]) + } + + /// One bit per lane for two adjacent quarters of compare results. + #[inline(always)] + fn bits16(lo: __m128i, hi: __m128i) -> u32 { + // SAFETY: SSE2 is compiled in (module docs); register-only ops. + unsafe { _mm_movemask_epi8(_mm_packs_epi16(lo, hi)) as u32 & 0xFFFF } + } + + #[inline(always)] + fn mask(self, other: Self, f: impl Fn(__m128i, __m128i) -> __m128i) -> u32 { + let a = self.quarters(); + let b = other.quarters(); + Self::bits16(f(a[0], b[0]), f(a[1], b[1])) | (Self::bits16(f(a[2], b[2]), f(a[3], b[3])) << 16) + } + + /// Per-lane unsigned minimum. + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + // SAFETY: SSE4.1 is compiled in (module docs); register-only op. + self.zip(other, |a, b| unsafe { _mm_min_epu16(a, b) }) + } + + /// Per-lane unsigned maximum. + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + // SAFETY: SSE4.1 is compiled in (module docs); register-only op. + self.zip(other, |a, b| unsafe { _mm_max_epu16(a, b) }) + } + + /// Per-lane addition clamped at `u16::MAX`. + #[inline(always)] + pub fn saturating_add(self, other: Self) -> Self { + // SAFETY: SSE2 is compiled in (module docs); register-only op. + self.zip(other, |a, b| unsafe { _mm_adds_epu16(a, b) }) + } + + /// Per-lane subtraction clamped at 0. + #[inline(always)] + pub fn saturating_sub(self, other: Self) -> Self { + // SAFETY: SSE2 is compiled in (module docs); register-only op. + self.zip(other, |a, b| unsafe { _mm_subs_epu16(a, b) }) + } + + /// Per-lane equality as a bitmask, lane `i` in bit `i`. + #[inline(always)] + pub fn cmpeq_mask(self, other: Self) -> u32 { + // SAFETY: SSE2 is compiled in (module docs); register-only op. + self.mask(other, |a, b| unsafe { _mm_cmpeq_epi16(a, b) }) + } + + /// Per-lane unsigned `self > other` as a bitmask, lane `i` in bit `i`. + #[inline(always)] + pub fn cmpgt_mask(self, other: Self) -> u32 { + // `a > b` unsigned is `max(a, b) == a` and `a != b`. + // SAFETY: SSE4.1 is compiled in (module docs); register-only ops. + self.mask(other, |a, b| unsafe { + _mm_andnot_si128(_mm_cmpeq_epi16(a, b), _mm_cmpeq_epi16(_mm_max_epu16(a, b), a)) + }) + } + + /// Smallest lane. + #[inline(always)] + pub fn reduce_min(self) -> u16 { + let q = self.quarters(); + // SAFETY: SSE4.1 is compiled in (module docs); register-only ops. + unsafe { + let m = _mm_min_epu16(_mm_min_epu16(q[0], q[1]), _mm_min_epu16(q[2], q[3])); + (_mm_cvtsi128_si32(_mm_minpos_epu16(m)) & 0xFFFF) as u16 + } + } + + /// Largest lane. + #[inline(always)] + pub fn reduce_max(self) -> u16 { + let q = self.quarters(); + // max(x) == !min(!x). + // SAFETY: SSE4.1 is compiled in (module docs); register-only ops. + unsafe { + let m = _mm_max_epu16(_mm_max_epu16(q[0], q[1]), _mm_max_epu16(q[2], q[3])); + let inv = _mm_xor_si128(m, _mm_set1_epi32(-1)); + !((_mm_cvtsi128_si32(_mm_minpos_epu16(inv)) & 0xFFFF) as u16) + } + } + + /// Lanes 0..16 converted to `f32`. + #[inline(always)] + pub fn to_f32x16_lo(self) -> F32x16 { + let q = self.quarters(); + Self::quarters_to_f32x16(q[0], q[1]) + } + + /// Lanes 16..32 converted to `f32`. + #[inline(always)] + pub fn to_f32x16_hi(self) -> F32x16 { + let q = self.quarters(); + Self::quarters_to_f32x16(q[2], q[3]) + } + + #[inline(always)] + fn quarters_to_f32x16(a: __m128i, b: __m128i) -> F32x16 { + // SAFETY: SSE4.1 and AVX are compiled in (module docs); register-only + // ops. `cvtepu16_epi32` widens the low four lanes; the high four are + // brought down by a whole-register 8-byte shift, which moves no byte + // within a lane. + unsafe { + let f = |v: __m128i| -> __m256 { + let lo = _mm_cvtepi32_ps(_mm_cvtepu16_epi32(v)); + let hi = _mm_cvtepi32_ps(_mm_cvtepu16_epi32(_mm_srli_si128::<8>(v))); + _mm256_set_m128(hi, lo) + }; + F32x16(F32x8(f(a)), F32x8(f(b))) + } + } +} diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 2818e681..939453f2 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -2687,6 +2687,239 @@ impl U16x32 { } } +impl U16x32 { + /// All 32 lanes zero. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// assert_eq!(U16x32::zero().to_array(), [0u16; 32]); + /// ``` + #[inline(always)] + pub fn zero() -> Self { + Self([0u16; 32]) + } +} + +/// The parts of the `U16x32` surface that need AVX2 integer instructions, +/// realized as two 256-bit halves (lanes 0..16 and 16..32). The +/// AVX-without-AVX2 arm supplies the same methods in `simd_avx.rs`. +#[cfg(not(all(target_feature = "avx", not(target_feature = "avx2"))))] +impl U16x32 { + /// The two 256-bit halves of the 64-byte array, loaded once. + #[inline(always)] + fn avx2_halves(self) -> (__m256i, __m256i) { + // SAFETY: AVX2 must be present at run time on any host this arm runs + // on (same footing as `U64x8::avx2_halves`). The array is 64 bytes, so + // both 32-byte unaligned loads are in bounds. + unsafe { + let p = self.0.as_ptr() as *const __m256i; + (_mm256_loadu_si256(p), _mm256_loadu_si256(p.add(1))) + } + } + + /// Store two 256-bit halves into a fresh `[u16; 32]`. + #[inline(always)] + fn from_avx2_halves(lo: __m256i, hi: __m256i) -> Self { + let mut o = [0u16; 32]; + // SAFETY: see `avx2_halves`; two 32-byte stores into a 64-byte array. + unsafe { + let p = o.as_mut_ptr() as *mut __m256i; + _mm256_storeu_si256(p, lo); + _mm256_storeu_si256(p.add(1), hi); + } + Self(o) + } + + /// One bit per u16 lane of a 256-bit compare result (all-ones or zero per + /// lane), lane 0 in bit 0. The pack folds each lane to one byte in lane + /// order because both inputs are 128-bit halves of the same register. + #[inline(always)] + fn lane_bits16(v: __m256i) -> u32 { + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { + let packed = _mm_packs_epi16(_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v)); + _mm_movemask_epi8(packed) as u32 & 0xFFFF + } + } + + /// Unsigned minimum of the eight u16 lanes of one 128-bit register. + #[inline(always)] + fn min_of_8(v: __m128i) -> u16 { + // SAFETY: SSE4.1 is implied by AVX2 (see `avx2_halves`). + unsafe { (_mm_cvtsi128_si32(_mm_minpos_epu16(v)) & 0xFFFF) as u16 } + } + + /// Per-lane unsigned minimum. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// let b = U16x32::splat(10); + /// assert_eq!(a.simd_min(b).to_array()[20], 10); + /// ``` + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + let (a0, a1) = self.avx2_halves(); + let (b0, b1) = other.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { Self::from_avx2_halves(_mm256_min_epu16(a0, b0), _mm256_min_epu16(a1, b1)) } + } + + /// Per-lane unsigned maximum. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.simd_max(U16x32::splat(10)).to_array()[3], 10); + /// ``` + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + let (a0, a1) = self.avx2_halves(); + let (b0, b1) = other.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { Self::from_avx2_halves(_mm256_max_epu16(a0, b0), _mm256_max_epu16(a1, b1)) } + } + + /// Per-lane addition clamped at `u16::MAX`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let s = U16x32::splat(u16::MAX - 1).saturating_add(U16x32::splat(5)); + /// assert_eq!(s.to_array(), [u16::MAX; 32]); + /// ``` + #[inline(always)] + pub fn saturating_add(self, other: Self) -> Self { + let (a0, a1) = self.avx2_halves(); + let (b0, b1) = other.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { Self::from_avx2_halves(_mm256_adds_epu16(a0, b0), _mm256_adds_epu16(a1, b1)) } + } + + /// Per-lane subtraction clamped at 0. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let s = U16x32::splat(3).saturating_sub(U16x32::splat(5)); + /// assert_eq!(s.to_array(), [0u16; 32]); + /// ``` + #[inline(always)] + pub fn saturating_sub(self, other: Self) -> Self { + let (a0, a1) = self.avx2_halves(); + let (b0, b1) = other.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { Self::from_avx2_halves(_mm256_subs_epu16(a0, b0), _mm256_subs_epu16(a1, b1)) } + } + + /// Per-lane equality as a bitmask: bit `i` is set iff lane `i` is equal. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.cmpeq_mask(U16x32::splat(5)), 1 << 5); + /// ``` + #[inline(always)] + pub fn cmpeq_mask(self, other: Self) -> u32 { + let (a0, a1) = self.avx2_halves(); + let (b0, b1) = other.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { Self::lane_bits16(_mm256_cmpeq_epi16(a0, b0)) | (Self::lane_bits16(_mm256_cmpeq_epi16(a1, b1)) << 16) } + } + + /// Per-lane unsigned `self > other` as a bitmask, lane `i` in bit `i`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.cmpgt_mask(U16x32::splat(29)), (1 << 30) | (1 << 31)); + /// assert_eq!(U16x32::splat(u16::MAX).cmpgt_mask(U16x32::splat(1)), u32::MAX); + /// ``` + #[inline(always)] + pub fn cmpgt_mask(self, other: Self) -> u32 { + let (a0, a1) = self.avx2_halves(); + let (b0, b1) = other.avx2_halves(); + // `a > b` unsigned is `max(a, b) == a` and `a != b`; no signed compare. + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { + let gt0 = _mm256_andnot_si256(_mm256_cmpeq_epi16(a0, b0), _mm256_cmpeq_epi16(_mm256_max_epu16(a0, b0), a0)); + let gt1 = _mm256_andnot_si256(_mm256_cmpeq_epi16(a1, b1), _mm256_cmpeq_epi16(_mm256_max_epu16(a1, b1), a1)); + Self::lane_bits16(gt0) | (Self::lane_bits16(gt1) << 16) + } + } + + /// Smallest lane. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| 100 - i as u16)); + /// assert_eq!(a.reduce_min(), 69); + /// ``` + #[inline(always)] + pub fn reduce_min(self) -> u16 { + let (a0, a1) = self.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { + let m = _mm256_min_epu16(a0, a1); + Self::min_of_8(_mm_min_epu16(_mm256_castsi256_si128(m), _mm256_extracti128_si256::<1>(m))) + } + } + + /// Largest lane. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 3)); + /// assert_eq!(a.reduce_max(), 93); + /// ``` + #[inline(always)] + pub fn reduce_max(self) -> u16 { + let (a0, a1) = self.avx2_halves(); + // max(x) == !min(!x); the complement keeps `minpos` as the one fold. + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { + let ones = _mm_set1_epi32(-1); + let m = _mm256_max_epu16(a0, a1); + let m = _mm_max_epu16(_mm256_castsi256_si128(m), _mm256_extracti128_si256::<1>(m)); + !Self::min_of_8(_mm_xor_si128(m, ones)) + } + } + + /// Lanes 0..16 converted to `f32`. Every `u16` is exact in `f32`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 1000)); + /// assert_eq!(a.to_f32x16_lo().to_array()[15], 15_000.0); + /// ``` + #[inline(always)] + pub fn to_f32x16_lo(self) -> F32x16 { + let (lo, _) = self.avx2_halves(); + Self::half_to_f32x16(lo) + } + + /// Lanes 16..32 converted to `f32`. Every `u16` is exact in `f32`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 1000)); + /// assert_eq!(a.to_f32x16_hi().to_array()[0], 16_000.0); + /// ``` + #[inline(always)] + pub fn to_f32x16_hi(self) -> F32x16 { + let (_, hi) = self.avx2_halves(); + Self::half_to_f32x16(hi) + } + + #[inline(always)] + fn half_to_f32x16(v: __m256i) -> F32x16 { + // SAFETY: AVX2 (see `avx2_halves`); register-only ops. + unsafe { + let a = _mm256_cvtepi32_ps(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(v))); + let b = _mm256_cvtepi32_ps(_mm256_cvtepu16_epi32(_mm256_extracti128_si256::<1>(v))); + F32x16(crate::simd_avx512::F32x8(a), crate::simd_avx512::F32x8(b)) + } + } +} + #[cfg(not(all(target_feature = "avx", not(target_feature = "avx2"))))] impl I32x16 { /// Store two 256-bit halves back into a fresh array (see `avx2_halves`). diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 641b4bf1..fba9cde4 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -1437,39 +1437,35 @@ impl U16x32 { }) } - /// Narrow back to u8 with unsigned saturation (32 × u16 → lower 32 bytes of U8x64). + /// Each lane clamped to 255 and narrowed to `u8`: bytes 0..32 come from + /// `self`, bytes 32..64 from `other`, both in lane order. #[inline(always)] pub fn pack_saturate_u8(self, other: Self) -> U8x64 { - // SAFETY: _mm512_packus_epi16 packs two __m512i of 16-bit into one __m512i of 8-bit. - U8x64(unsafe { _mm512_packus_epi16(self.0, other.0) }) + // SAFETY: AVX-512BW (see `reduce_sum`); register-only ops. + // `cvtusepi16_epi8` narrows with unsigned saturation and keeps lane + // order, unlike `packus_epi16`, which saturates as signed and + // interleaves its two inputs per 128-bit lane. + U8x64(unsafe { + let lo = _mm512_cvtusepi16_epi8(self.0); + let hi = _mm512_cvtusepi16_epi8(other.0); + _mm512_inserti64x4::<1>(_mm512_castsi256_si512(lo), hi) + }) } /// Shift right each 16-bit lane by immediate. #[inline(always)] pub fn shr(self, imm: u32) -> Self { - Self(unsafe { - match imm { - 1 => _mm512_srli_epi16(self.0, 1), - 2 => _mm512_srli_epi16(self.0, 2), - 4 => _mm512_srli_epi16(self.0, 4), - 8 => _mm512_srli_epi16(self.0, 8), - _ => _mm512_setzero_si512(), - } - }) + // SAFETY: AVX-512BW (see `reduce_sum`); register-only ops. A count of + // 16 or more gives 0, which is what `srl_epi16` does with it. + Self(unsafe { _mm512_srl_epi16(self.0, _mm_cvtsi64_si128(imm as i64)) }) } /// Shift left each 16-bit lane by immediate. #[inline(always)] pub fn shl(self, imm: u32) -> Self { - Self(unsafe { - match imm { - 1 => _mm512_slli_epi16(self.0, 1), - 2 => _mm512_slli_epi16(self.0, 2), - 4 => _mm512_slli_epi16(self.0, 4), - 8 => _mm512_slli_epi16(self.0, 8), - _ => _mm512_setzero_si512(), - } - }) + // SAFETY: AVX-512BW (see `reduce_sum`); register-only ops. A count of + // 16 or more gives 0, which is what `sll_epi16` does with it. + Self(unsafe { _mm512_sll_epi16(self.0, _mm_cvtsi64_si128(imm as i64)) }) } /// Multiply and keep low 16 bits (wrapping). @@ -1479,11 +1475,207 @@ impl U16x32 { Self(unsafe { _mm512_mullo_epi16(self.0, other.0) }) } - /// Horizontal sum of all 32 lanes. + /// Wrapping horizontal sum of all 32 lanes (the result wraps modulo + /// 2^16, as every other backend's `U16x32::reduce_sum` does). + /// + /// ``` + /// use ndarray::simd::U16x32; + /// assert_eq!(U16x32::splat(3).reduce_sum(), 96); + /// assert_eq!(U16x32::splat(u16::MAX).reduce_sum(), u16::MAX.wrapping_mul(32)); + /// ``` #[inline(always)] - pub fn reduce_sum(self) -> u32 { - let arr = self.to_array(); - arr.iter().map(|&v| v as u32).sum() + pub fn reduce_sum(self) -> u16 { + // SAFETY: AVX-512F/BW are present whenever this type is the exported + // `U16x32` (the `avx512f` arm); register-only ops. + unsafe { + let lo = _mm512_cvtepu16_epi32(_mm512_castsi512_si256(self.0)); + let hi = _mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64::<1>(self.0)); + _mm512_reduce_add_epi32(_mm512_add_epi32(lo, hi)) as u16 + } + } + + /// Per-lane unsigned minimum. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.simd_min(U16x32::splat(10)).to_array()[20], 10); + /// ``` + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + // SAFETY: AVX-512BW (see `reduce_sum`); register-only op. + Self(unsafe { _mm512_min_epu16(self.0, other.0) }) + } + + /// Per-lane unsigned maximum. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.simd_max(U16x32::splat(10)).to_array()[3], 10); + /// ``` + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + // SAFETY: AVX-512BW (see `reduce_sum`); register-only op. + Self(unsafe { _mm512_max_epu16(self.0, other.0) }) + } + + /// Per-lane addition clamped at `u16::MAX`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let s = U16x32::splat(u16::MAX - 1).saturating_add(U16x32::splat(5)); + /// assert_eq!(s.to_array(), [u16::MAX; 32]); + /// ``` + #[inline(always)] + pub fn saturating_add(self, other: Self) -> Self { + // SAFETY: AVX-512BW (see `reduce_sum`); register-only op. + Self(unsafe { _mm512_adds_epu16(self.0, other.0) }) + } + + /// Per-lane subtraction clamped at 0. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// assert_eq!(U16x32::splat(3).saturating_sub(U16x32::splat(5)).to_array(), [0u16; 32]); + /// ``` + #[inline(always)] + pub fn saturating_sub(self, other: Self) -> Self { + // SAFETY: AVX-512BW (see `reduce_sum`); register-only op. + Self(unsafe { _mm512_subs_epu16(self.0, other.0) }) + } + + /// Per-lane equality as a bitmask: bit `i` is set iff lane `i` is equal. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.cmpeq_mask(U16x32::splat(5)), 1 << 5); + /// ``` + #[inline(always)] + pub fn cmpeq_mask(self, other: Self) -> u32 { + // SAFETY: AVX-512BW (see `reduce_sum`); register-only op. + unsafe { _mm512_cmpeq_epu16_mask(self.0, other.0) } + } + + /// Per-lane unsigned `self > other` as a bitmask, lane `i` in bit `i`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.cmpgt_mask(U16x32::splat(29)), (1 << 30) | (1 << 31)); + /// ``` + #[inline(always)] + pub fn cmpgt_mask(self, other: Self) -> u32 { + // SAFETY: AVX-512BW (see `reduce_sum`); register-only op. + unsafe { _mm512_cmpgt_epu16_mask(self.0, other.0) } + } + + /// Smallest lane. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| 100 - i as u16)); + /// assert_eq!(a.reduce_min(), 69); + /// ``` + #[inline(always)] + pub fn reduce_min(self) -> u16 { + // SAFETY: AVX-512BW (see `reduce_sum`); SSE4.1 is implied. Register-only ops. + unsafe { + let m = _mm256_min_epu16(_mm512_castsi512_si256(self.0), _mm512_extracti64x4_epi64::<1>(self.0)); + let m = _mm_min_epu16(_mm256_castsi256_si128(m), _mm256_extracti128_si256::<1>(m)); + (_mm_cvtsi128_si32(_mm_minpos_epu16(m)) & 0xFFFF) as u16 + } + } + + /// Largest lane. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 3)); + /// assert_eq!(a.reduce_max(), 93); + /// ``` + #[inline(always)] + pub fn reduce_max(self) -> u16 { + // max(x) == !min(!x). + // SAFETY: AVX-512BW (see `reduce_sum`); SSE4.1 is implied. Register-only ops. + unsafe { + let m = _mm256_max_epu16(_mm512_castsi512_si256(self.0), _mm512_extracti64x4_epi64::<1>(self.0)); + let m = _mm_max_epu16(_mm256_castsi256_si128(m), _mm256_extracti128_si256::<1>(m)); + !((_mm_cvtsi128_si32(_mm_minpos_epu16(_mm_xor_si128(m, _mm_set1_epi32(-1)))) & 0xFFFF) as u16) + } + } + + /// Lanes 0..16 converted to `f32`. Every `u16` is exact in `f32`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 1000)); + /// assert_eq!(a.to_f32x16_lo().to_array()[15], 15_000.0); + /// ``` + #[inline(always)] + pub fn to_f32x16_lo(self) -> F32x16 { + // SAFETY: AVX-512F/BW (see `reduce_sum`); register-only ops. + F32x16(unsafe { _mm512_cvtepi32_ps(_mm512_cvtepu16_epi32(_mm512_castsi512_si256(self.0))) }) + } + + /// Lanes 16..32 converted to `f32`. Every `u16` is exact in `f32`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 1000)); + /// assert_eq!(a.to_f32x16_hi().to_array()[0], 16_000.0); + /// ``` + #[inline(always)] + pub fn to_f32x16_hi(self) -> F32x16 { + // SAFETY: AVX-512F/BW (see `reduce_sum`); register-only ops. + F32x16(unsafe { _mm512_cvtepi32_ps(_mm512_cvtepu16_epi32(_mm512_extracti64x4_epi64::<1>(self.0))) }) + } +} + +impl Default for U16x32 { + #[inline(always)] + fn default() -> Self { + Self::zero() + } +} + +macro_rules! u16x32_bitop { + ($tr:ident, $f:ident, $atr:ident, $af:ident, $intr:ident) => { + impl $tr for U16x32 { + type Output = Self; + #[inline(always)] + fn $f(self, rhs: Self) -> Self { + // SAFETY: AVX-512F (see `U16x32::reduce_sum`); register-only op. + Self(unsafe { $intr(self.0, rhs.0) }) + } + } + impl $atr for U16x32 { + #[inline(always)] + fn $af(&mut self, rhs: Self) { + *self = $tr::$f(*self, rhs); + } + } + }; +} +u16x32_bitop!(BitAnd, bitand, BitAndAssign, bitand_assign, _mm512_and_si512); +u16x32_bitop!(BitOr, bitor, BitOrAssign, bitor_assign, _mm512_or_si512); +u16x32_bitop!(BitXor, bitxor, BitXorAssign, bitxor_assign, _mm512_xor_si512); + +impl Not for U16x32 { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + // SAFETY: AVX-512F (see `U16x32::reduce_sum`); register-only op. + Self(unsafe { _mm512_xor_si512(self.0, _mm512_set1_epi32(-1)) }) + } +} + +impl SubAssign for U16x32 { + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + // SAFETY: AVX-512BW (see `U16x32::reduce_sum`); register-only op. + self.0 = unsafe { _mm512_sub_epi16(self.0, rhs.0) }; } } diff --git a/src/simd_nightly/u_word_types.rs b/src/simd_nightly/u_word_types.rs index 78a787bd..f58f40f9 100644 --- a/src/simd_nightly/u_word_types.rs +++ b/src/simd_nightly/u_word_types.rs @@ -899,6 +899,75 @@ impl U16x32 { pub fn cmpgt_mask(self, other: Self) -> u32 { self.0.simd_gt(other.0).to_bitmask() as u32 } + + // ── Facade surface shared with the other backends ───────────── + + /// All 32 lanes zero. + #[inline(always)] + pub fn zero() -> Self { + Self(u16x32::splat(0)) + } + + /// Bytes 0..32 of `v`, zero-extended to `u16`. + #[inline(always)] + pub fn from_u8x64_lo(v: super::u8_types::U8x64) -> Self { + Self(core::simd::u8x32::from_slice(&v.0.as_array()[..32]).cast::()) + } + + /// Bytes 32..64 of `v`, zero-extended to `u16`. + #[inline(always)] + pub fn from_u8x64_hi(v: super::u8_types::U8x64) -> Self { + Self(core::simd::u8x32::from_slice(&v.0.as_array()[32..]).cast::()) + } + + /// `self` then `other`, each lane clamped to 255 and narrowed to `u8`: + /// bytes 0..32 come from `self`, bytes 32..64 from `other`. + #[inline(always)] + pub fn pack_saturate_u8(self, other: Self) -> super::u8_types::U8x64 { + let cap = u16x32::splat(255); + let a = self.0.simd_min(cap).cast::(); + let b = other.0.simd_min(cap).cast::(); + let mut out = [0u8; 64]; + out[..32].copy_from_slice(a.as_array()); + out[32..].copy_from_slice(b.as_array()); + super::u8_types::U8x64(core::simd::u8x64::from_array(out)) + } + + /// Logical right shift of every lane; a count of 16 or more gives 0. + #[inline(always)] + pub fn shr(self, imm: u32) -> Self { + if imm >= 16 { + return Self::zero(); + } + Self(self.0 >> u16x32::splat(imm as u16)) + } + + /// Left shift of every lane; a count of 16 or more gives 0. + #[inline(always)] + pub fn shl(self, imm: u32) -> Self { + if imm >= 16 { + return Self::zero(); + } + Self(self.0 << u16x32::splat(imm as u16)) + } + + /// Per-lane wrapping multiply (low 16 bits of the product). + #[inline(always)] + pub fn mullo(self, other: Self) -> Self { + Self(self.0 * other.0) + } + + /// Lanes 0..16 converted to `f32`. Every `u16` is exact in `f32`. + #[inline(always)] + pub fn to_f32x16_lo(self) -> super::f32_types::F32x16 { + super::f32_types::F32x16(u16x16::from_slice(&self.0.as_array()[..16]).cast::()) + } + + /// Lanes 16..32 converted to `f32`. Every `u16` is exact in `f32`. + #[inline(always)] + pub fn to_f32x16_hi(self) -> super::f32_types::F32x16 { + super::f32_types::F32x16(u16x16::from_slice(&self.0.as_array()[16..]).cast::()) + } } impl Default for U16x32 { diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index c7fd501b..09d2733c 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -981,6 +981,136 @@ impl U16x32 { } Self(out) } + + /// All 32 lanes zero. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// assert_eq!(U16x32::zero().to_array(), [0u16; 32]); + /// ``` + #[inline(always)] + pub fn zero() -> Self { + Self([0u16; 32]) + } + + /// Per-lane unsigned minimum. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.simd_min(U16x32::splat(10)).to_array()[20], 10); + /// ``` + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + Self(core::array::from_fn(|i| self.0[i].min(other.0[i]))) + } + + /// Per-lane unsigned maximum. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.simd_max(U16x32::splat(10)).to_array()[3], 10); + /// ``` + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + Self(core::array::from_fn(|i| self.0[i].max(other.0[i]))) + } + + /// Per-lane addition clamped at `u16::MAX`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let s = U16x32::splat(u16::MAX - 1).saturating_add(U16x32::splat(5)); + /// assert_eq!(s.to_array(), [u16::MAX; 32]); + /// ``` + #[inline(always)] + pub fn saturating_add(self, other: Self) -> Self { + Self(core::array::from_fn(|i| self.0[i].saturating_add(other.0[i]))) + } + + /// Per-lane subtraction clamped at 0. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// assert_eq!(U16x32::splat(3).saturating_sub(U16x32::splat(5)).to_array(), [0u16; 32]); + /// ``` + #[inline(always)] + pub fn saturating_sub(self, other: Self) -> Self { + Self(core::array::from_fn(|i| self.0[i].saturating_sub(other.0[i]))) + } + + /// Per-lane equality as a bitmask: bit `i` is set iff lane `i` is equal. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.cmpeq_mask(U16x32::splat(5)), 1 << 5); + /// ``` + #[inline(always)] + pub fn cmpeq_mask(self, other: Self) -> u32 { + (0..32).fold(0u32, |m, i| m | (((self.0[i] == other.0[i]) as u32) << i)) + } + + /// Per-lane unsigned `self > other` as a bitmask, lane `i` in bit `i`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16)); + /// assert_eq!(a.cmpgt_mask(U16x32::splat(29)), (1 << 30) | (1 << 31)); + /// ``` + #[inline(always)] + pub fn cmpgt_mask(self, other: Self) -> u32 { + (0..32).fold(0u32, |m, i| m | (((self.0[i] > other.0[i]) as u32) << i)) + } + + /// Smallest lane. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| 100 - i as u16)); + /// assert_eq!(a.reduce_min(), 69); + /// ``` + #[inline(always)] + pub fn reduce_min(self) -> u16 { + self.0.iter().copied().fold(u16::MAX, u16::min) + } + + /// Largest lane. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 3)); + /// assert_eq!(a.reduce_max(), 93); + /// ``` + #[inline(always)] + pub fn reduce_max(self) -> u16 { + self.0.iter().copied().fold(0, u16::max) + } + + /// Lanes 0..16 converted to `f32`. Every `u16` is exact in `f32`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 1000)); + /// assert_eq!(a.to_f32x16_lo().to_array()[15], 15_000.0); + /// ``` + #[inline(always)] + pub fn to_f32x16_lo(self) -> F32x16 { + F32x16(core::array::from_fn(|i| self.0[i] as f32)) + } + + /// Lanes 16..32 converted to `f32`. Every `u16` is exact in `f32`. + /// + /// ``` + /// use ndarray::simd::U16x32; + /// let a = U16x32::from_array(core::array::from_fn(|i| i as u16 * 1000)); + /// assert_eq!(a.to_f32x16_hi().to_array()[0], 16_000.0); + /// ``` + #[inline(always)] + pub fn to_f32x16_hi(self) -> F32x16 { + F32x16(core::array::from_fn(|i| self.0[16 + i] as f32)) + } } // Extra methods for I32x16 that float types have via the macro From ca7dbab07be95646c5f0c7c302c1b1b558f2bdd0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 09:06:50 +0000 Subject: [PATCH 2/2] Record the U16x32 facade work and the clippy --config trap on the blackboard Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XAqKMMjKnig6MXTd41Lf2v --- .claude/blackboard.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index c91a0a81..f8f21b17 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -3827,3 +3827,25 @@ back into clippy::all), fmt clean. G5 asm identity: generic == hand-written on x negative control differs. G6 2% timing band not resolvable here (identical code varies up to 17%); recorded as NOT MET, G5 is the evidence. Results in plan §11. Next: phase (b) SIMD primitives in src/simd_*.rs under W1a. + +## U16x32 facade gaps filled on every backend (2026-10-29, branch claude/u16x32-facade-gaps) + +Trigger: turbovec's AVX2 FastScan kernel is being rewritten against `ndarray::simd` at 64-byte width +(U8x64 -> U16x32 -> F32x16). U16x32 had a different method set per backend and no u16 -> f32 path. + +Added on AVX-512, AVX2, AVX (simd_avx.rs, 128-bit quarters), scalar and nightly: `zero`, `simd_min/max`, +`saturating_add/sub`, `cmpeq_mask`/`cmpgt_mask` (u32), `reduce_min/max`, `to_f32x16_lo/hi`; on AVX-512 the bit +operators, `Not`, `SubAssign`, `Default`; on nightly the widening/shift/mullo/pack methods. `reduce_sum` returns +`u16` (wrapping) everywhere; AVX-512 returned `u32`. + +Bugs found by the new parity test (`simd::u16x32_facade_parity`), both pre-existing on AVX-512 and now fixed: +`shl`/`shr` returned 0 for every count except 1, 2, 4, 8 (including 0); `pack_saturate_u8` used `packus_epi16`, +which saturates as signed and interleaves its inputs per 128-bit lane. + +Witnessed: parity 4/4 on AVX-512 (native), AVX2 (config-v3), AVX (config-avx, qemu SandyBridge), scalar (config-v2, +qemu Nehalem) and nightly-simd. Lib suite 2653 passed. clippy -D warnings clean on default/v3/v2; the AVX arm's +clippy errors are all pre-existing (same set on master). Disable run: AVX2 cmpgt as `>=` -> parity red. + +Trap recorded: `cargo --config clippy` DROPS the config (clippy is an external subcommand); the v3 lint pass +reported clean in 0.05s while linting the native build. Put `--config` after the subcommand: +`cargo clippy --config .cargo/config-v3.toml`. `cargo --config test` does apply it.