From fb595c98fec66efa3c5271599566bab38f03b1f9 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Mon, 9 Nov 2026 01:14:24 +0900 Subject: [PATCH 1/2] perf(h264): hoist SAD dispatch out of score rows Score rows validate the block width and resolve the SAD arithmetic once per block, then call the safe per-row method. Row slices, every row cutoff and equal-score ties are unchanged. Signed-off-by: Gunwoo Kim --- crates/coreav-h264-encoder/src/inter.rs | 126 +++++++++++++++++++++- crates/coreav-h264-encoder/src/kernels.rs | 69 ++++++++++-- 2 files changed, 184 insertions(+), 11 deletions(-) diff --git a/crates/coreav-h264-encoder/src/inter.rs b/crates/coreav-h264-encoder/src/inter.rs index fb5826977..f9818c954 100644 --- a/crates/coreav-h264-encoder/src/inter.rs +++ b/crates/coreav-h264-encoder/src/inter.rs @@ -149,12 +149,10 @@ fn score_rows( motion_cost: u64, cutoff: Option, ) -> Result> { + let sad = crate::kernels::Sad::new(width)?; let mut score = motion_cost; for (original, predicted) in source.chunks(STRIDE).zip(predicted.chunks(stride)) { - score += crate::kernels::sad( - original.get(..width).ok_or(Error::Internal)?, - predicted.get(..width).ok_or(Error::Internal)?, - )?; + score += sad.row(original, predicted)?; if cutoff.is_some_and(|limit| score > limit) { return Ok(None); } @@ -1928,6 +1926,126 @@ mod tests { assert!(motions.ranked.contains(&Some((16, 8, mv)))); } + fn per_row_scores( + source: &[u8], + predicted: &[u8], + stride: usize, + width: usize, + motion_cost: u64, + cutoff: Option, + ) -> Result> { + let mut score = motion_cost; + for (original, predicted) in source.chunks(STRIDE).zip(predicted.chunks(stride)) { + score += crate::kernels::sad( + original.get(..width).ok_or(Error::Internal)?, + predicted.get(..width).ok_or(Error::Internal)?, + )?; + if cutoff.is_some_and(|limit| score > limit) { + return Ok(None); + } + } + Ok(Some(score)) + } + + #[test] + fn block_sad_dispatch_matches_per_row_scores_cutoffs_and_failures() { + let mut seed = 0x5ad0_d15b_a7c4_e001u64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut compared = 0; + for width in 0..=20 { + for height in 1..=16 { + for stride in [width.max(1), 16, 21, STRIDE, 64] { + let extremes = next() % 4 == 0; + let mut sample = || { + let value = next(); + if extremes { + [0, 255][value as usize % 2] + } else { + value as u8 + } + }; + let source: Vec = (0..(height - 1) * STRIDE + width.max(1)) + .map(|_| sample()) + .collect(); + let predicted: Vec = (0..(height - 1) * stride + width.max(1) + 3) + .map(|_| sample()) + .collect(); + let motion_cost = next() % 512; + let mut prefixes = vec![motion_cost]; + if width <= 16 { + for (row, original) in source.chunks(STRIDE).enumerate() { + let start = row * stride; + let total = original + .iter() + .take(width) + .zip(&predicted[start..start + width]) + .map(|(&a, &b)| u64::from(a.abs_diff(b))) + .sum::(); + prefixes.push(prefixes.last().unwrap() + total); + } + assert_eq!( + score_rows(&source, &predicted, stride, width, motion_cost, None), + Ok(Some(*prefixes.last().unwrap())) + ); + } + let mut cutoffs = vec![None, Some(0), Some(u64::MAX)]; + for &prefix in &prefixes { + cutoffs.extend([prefix.checked_sub(1), Some(prefix), Some(prefix + 1)]); + } + let short = (height - 1) * stride + width.max(1) - 1; + for cutoff in cutoffs { + for (source, predicted) in [ + (&source[..], &predicted[..]), + (&source[..source.len() - 1], &predicted[..]), + (&source[..], &predicted[..short]), + ] { + assert_eq!( + score_rows(source, predicted, stride, width, motion_cost, cutoff), + per_row_scores( + source, + predicted, + stride, + width, + motion_cost, + cutoff + ), + "width {width} height {height} stride {stride} cutoff {cutoff:?}" + ); + compared += 1; + } + } + } + } + } + assert!(compared > 50_000); + } + + #[test] + fn block_sad_keeps_row_cutoffs_equal_score_ties_and_admission() { + let source = [10u8; STRIDE * 3 + 16]; + let mut predicted = [0u8; 16 * 4]; + let rows = |predicted: &[u8], cutoff| score_rows(&source, predicted, 16, 16, 5, cutoff); + assert_eq!(rows(&predicted, None), Ok(Some(645))); + assert_eq!(rows(&predicted, Some(645)), Ok(Some(645))); + assert_eq!(rows(&predicted, Some(644)), Ok(None)); + assert_eq!(rows(&predicted, Some(165)), Ok(None)); + assert_eq!(rows(&predicted, Some(4)), Ok(None)); + assert_eq!(rows(&predicted[..15], Some(0)), Err(Error::Internal)); + predicted[16 * 3..].fill(10); + assert_eq!(rows(&predicted, Some(485)), Ok(Some(485))); + assert_eq!(rows(&predicted, Some(484)), Ok(None)); + assert_eq!( + score_rows(&source, &predicted, 16, 17, 5, None), + Err(Error::Internal) + ); + assert_eq!(score_rows(&[], &[], 16, 16, 7, Some(0)), Ok(Some(7))); + } + #[test] fn cheap_finalists_keep_protected_seeds_and_deterministic_ties() { let mut motions = Motions::new(); diff --git a/crates/coreav-h264-encoder/src/kernels.rs b/crates/coreav-h264-encoder/src/kernels.rs index 4a0b45271..96b2b3787 100644 --- a/crates/coreav-h264-encoder/src/kernels.rs +++ b/crates/coreav-h264-encoder/src/kernels.rs @@ -16,19 +16,43 @@ pub(crate) fn pixels() -> Auto { } pub(crate) fn sad(a: &[u8], b: &[u8]) -> Result { - if a.len() != b.len() || a.len() > 16 { + if a.len() != b.len() { return Err(Error::Internal); } + Sad::new(a.len())?.row(a, b) +} + +#[derive(Clone, Copy)] +pub(crate) struct Sad { + width: usize, #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] - { - if let Some(simd) = arithmetic() { + simd: Option, +} + +impl Sad { + pub(crate) fn new(width: usize) -> Result { + if width > 16 { + return Err(Error::Internal); + } + Ok(Self { + width, + #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] + simd: arithmetic(), + }) + } + + pub(crate) fn row(self, a: &[u8], b: &[u8]) -> Result { + let a = a.get(..self.width).ok_or(Error::Internal)?; + let b = b.get(..self.width).ok_or(Error::Internal)?; + #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] + if let Some(simd) = self.simd { return Ok(simd.sad(a, b)); } + Ok(a.iter() + .zip(b) + .map(|(&a, &b)| u64::from(a.abs_diff(b))) + .sum()) } - Ok(a.iter() - .zip(b) - .map(|(&a, &b)| u64::from(a.abs_diff(b))) - .sum()) } #[cfg(all(target_arch = "x86_64", not(feature = "scalar")))] @@ -237,6 +261,37 @@ mod tests { assert_eq!(sad(&[0; 16], &[0; 15]), Err(Error::Internal)); } + #[test] + fn block_sad_admits_width_once_and_reads_exactly_width_bytes_per_row() { + assert!(Sad::new(17).is_err()); + let mut seed = 0x3c6e_f372_fe94_f82bu64; + for width in 0..=16 { + let block = Sad::new(width).unwrap(); + for _ in 0..256 { + let mut a = [0; 24]; + let mut b = [0; 24]; + for v in a.iter_mut().chain(&mut b) { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + *v = seed as u8; + } + let want = a[..width] + .iter() + .zip(&b[..width]) + .map(|(&a, &b)| u64::from(a.abs_diff(b))) + .sum(); + assert_eq!(block.row(&a, &b), Ok(want)); + assert_eq!(block.row(&a[..width], &b[..width]), Ok(want)); + assert_eq!(sad(&a[..width], &b[..width]), Ok(want)); + if width > 0 { + assert_eq!(block.row(&a[..width - 1], &b), Err(Error::Internal)); + assert_eq!(block.row(&a, &b[..width - 1]), Err(Error::Internal)); + } + } + } + } + #[test] fn scalar_feature_overrides_pixel_dispatch() { #[cfg(feature = "scalar")] From fba8f678d1a5f2214e6aa1ece1f194cab356711d Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Mon, 9 Nov 2026 01:42:50 +0900 Subject: [PATCH 2/2] perf(h264): inline the per-row SAD call Signed-off-by: Gunwoo Kim --- crates/coreav-h264-encoder/src/kernels.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/coreav-h264-encoder/src/kernels.rs b/crates/coreav-h264-encoder/src/kernels.rs index 96b2b3787..a3cfa2b7d 100644 --- a/crates/coreav-h264-encoder/src/kernels.rs +++ b/crates/coreav-h264-encoder/src/kernels.rs @@ -30,6 +30,7 @@ pub(crate) struct Sad { } impl Sad { + #[inline] pub(crate) fn new(width: usize) -> Result { if width > 16 { return Err(Error::Internal); @@ -41,6 +42,7 @@ impl Sad { }) } + #[inline] pub(crate) fn row(self, a: &[u8], b: &[u8]) -> Result { let a = a.get(..self.width).ok_or(Error::Internal)?; let b = b.get(..self.width).ok_or(Error::Internal)?;