perf(theta,tuple): add inlining hints to codec, hashing and error paths

CHANGELOG.md

@@ -19,6 +19,7 @@ All significant changes to this project will be documented in this file.
 * Repeated scalar `TDigest::rank` queries are faster, including small digests and data with repeated values.
 * `TDigest` and `TDigestMut` CDF and PMF batches are faster for many split points; batches with sparse split points keep the original binary-search behavior.
 * `BloomFilter::insert` is faster for streams of new items when multiple hashes are configured.
+* Tuple sketch serialization and deserialization are faster.
 
 ## v0.6.0 (2026-10-29)
 

datasketches/src/codec/decode.rs

@@ -26,13 +26,15 @@ pub struct SketchSlice<'a> {
 
 impl SketchSlice<'_> {
     /// Creates a new `SketchSlice` from the given byte slice.
+    #[inline]
     pub fn new(slice: &[u8]) -> SketchSlice<'_> {
         SketchSlice {
             slice: Cursor::new(slice),
         }
     }
 
     /// Advances the position of the slice by `n` bytes.
+    #[inline]
     pub fn advance(&mut self, n: u64) {
         let pos = self.slice.position();
         self.slice.set_position(pos + n);
@@ -42,137 +44,157 @@ impl SketchSlice<'_> {
     ///
     /// Useful for handing the remaining bytes to a variable-length decoder that reports how many
     /// bytes it consumed; pair it with [`advance`](Self::advance).
+    #[inline]
     pub fn remaining(&self) -> &[u8] {
         let buf = self.slice.get_ref();
         let pos = (self.slice.position() as usize).min(buf.len());
         &buf[pos..]
     }
 
     /// Reads exactly `buf.len()` bytes from the slice into `buf`.
+    #[inline]
     pub fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
         self.slice.read_exact(buf)
     }
 
     /// Reads a single byte from the slice and returns it as a `u8`.
+    #[inline]
     pub fn read_u8(&mut self) -> io::Result<u8> {
         let mut buf = [0u8; 1];
         self.read_exact(&mut buf)?;
         Ok(buf[0])
     }
 
     /// Reads a single byte from the slice and returns it as an `i8`.
+    #[inline]
     pub fn read_i8(&mut self) -> io::Result<i8> {
         let mut buf = [0u8; 1];
         self.read_exact(&mut buf)?;
         Ok(buf[0] as i8)
     }
 
     /// Reads a 16-bit unsigned integer from the slice in little-endian byte order.
+    #[inline]
     pub fn read_u16_le(&mut self) -> io::Result<u16> {
         let mut buf = [0u8; 2];
         self.read_exact(&mut buf)?;
         Ok(u16::from_le_bytes(buf))
     }
 
     /// Reads a 16-bit unsigned integer from the slice in big-endian byte order.
+    #[inline]
     pub fn read_u16_be(&mut self) -> io::Result<u16> {
         let mut buf = [0u8; 2];
         self.read_exact(&mut buf)?;
         Ok(u16::from_be_bytes(buf))
     }
 
     /// Reads a 16-bit signed integer from the slice in little-endian byte order.
+    #[inline]
     pub fn read_i16_le(&mut self) -> io::Result<i16> {
         let mut buf = [0u8; 2];
         self.read_exact(&mut buf)?;
         Ok(i16::from_le_bytes(buf))
     }
 
     /// Reads a 16-bit signed integer from the slice in big-endian byte order.
+    #[inline]
     pub fn read_i16_be(&mut self) -> io::Result<i16> {
         let mut buf = [0u8; 2];
         self.read_exact(&mut buf)?;
         Ok(i16::from_be_bytes(buf))
     }
 
     /// Reads a 32-bit unsigned integer from the slice in little-endian byte order.
+    #[inline]
     pub fn read_u32_le(&mut self) -> io::Result<u32> {
         let mut buf = [0u8; 4];
         self.read_exact(&mut buf)?;
         Ok(u32::from_le_bytes(buf))
     }
 
     /// Reads a 32-bit unsigned integer from the slice in big-endian byte order.
+    #[inline]
     pub fn read_u32_be(&mut self) -> io::Result<u32> {
         let mut buf = [0u8; 4];
         self.read_exact(&mut buf)?;
         Ok(u32::from_be_bytes(buf))
     }
 
     /// Reads a 32-bit signed integer from the slice in little-endian byte order.
+    #[inline]
     pub fn read_i32_le(&mut self) -> io::Result<i32> {
         let mut buf = [0u8; 4];
         self.read_exact(&mut buf)?;
         Ok(i32::from_le_bytes(buf))
     }
 
     /// Reads a 32-bit signed integer from the slice in big-endian byte order.
+    #[inline]
     pub fn read_i32_be(&mut self) -> io::Result<i32> {
         let mut buf = [0u8; 4];
         self.read_exact(&mut buf)?;
         Ok(i32::from_be_bytes(buf))
     }
 
     /// Reads a 64-bit unsigned integer from the slice in little-endian byte order.
+    #[inline]
     pub fn read_u64_le(&mut self) -> io::Result<u64> {
         let mut buf = [0u8; 8];
         self.read_exact(&mut buf)?;
         Ok(u64::from_le_bytes(buf))
     }
 
     /// Reads a 64-bit unsigned integer from the slice in big-endian byte order.
+    #[inline]
     pub fn read_u64_be(&mut self) -> io::Result<u64> {
         let mut buf = [0u8; 8];
         self.read_exact(&mut buf)?;
         Ok(u64::from_be_bytes(buf))
     }
 
     /// Reads a 64-bit signed integer from the slice in little-endian byte order.
+    #[inline]
     pub fn read_i64_le(&mut self) -> io::Result<i64> {
         let mut buf = [0u8; 8];
         self.read_exact(&mut buf)?;
         Ok(i64::from_le_bytes(buf))
     }
 
     /// Reads a 64-bit signed integer from the slice in big-endian byte order.
+    #[inline]
     pub fn read_i64_be(&mut self) -> io::Result<i64> {
         let mut buf = [0u8; 8];
         self.read_exact(&mut buf)?;
         Ok(i64::from_be_bytes(buf))
     }
 
     /// Reads a 32-bit floating-point number from the slice in little-endian byte order.
+    #[inline]
     pub fn read_f32_le(&mut self) -> io::Result<f32> {
         let mut buf = [0u8; 4];
         self.read_exact(&mut buf)?;
         Ok(f32::from_le_bytes(buf))
     }
 
     /// Reads a 32-bit floating-point number from the slice in big-endian byte order.
+    #[inline]
     pub fn read_f32_be(&mut self) -> io::Result<f32> {
         let mut buf = [0u8; 4];
         self.read_exact(&mut buf)?;
         Ok(f32::from_be_bytes(buf))
     }
 
     /// Reads a 64-bit floating-point number from the slice in little-endian byte order.
+    #[inline]
     pub fn read_f64_le(&mut self) -> io::Result<f64> {
         let mut buf = [0u8; 8];
         self.read_exact(&mut buf)?;
         Ok(f64::from_le_bytes(buf))
     }
 
     /// Reads a 64-bit floating-point number from the slice in big-endian byte order.
+    #[inline]
     pub fn read_f64_be(&mut self) -> io::Result<f64> {
         let mut buf = [0u8; 8];
         self.read_exact(&mut buf)?;

datasketches/src/codec/encode.rs

@@ -22,108 +22,129 @@ pub struct SketchBytes {
 
 impl SketchBytes {
     /// Creates an empty `SketchBytes` with at least the specified capacity.
+    #[inline]
     pub fn with_capacity(capacity: usize) -> Self {
         Self {
             bytes: Vec::with_capacity(capacity),
         }
     }
 
     /// Returns the underlying `Vec<u8>`, consuming the `SketchBytes`.
+    #[inline]
     pub fn into_bytes(self) -> Vec<u8> {
         self.bytes
     }
 
     /// Writes the given byte slice to the `SketchBytes`.
+    #[inline]
     pub fn write(&mut self, buf: &[u8]) {
         self.bytes.extend_from_slice(buf);
     }
 
     /// Writes a single byte to the `SketchBytes`.
+    #[inline]
     pub fn write_u8(&mut self, n: u8) {
         self.bytes.push(n);
     }
 
     /// Writes a single byte to the `SketchBytes`.
+    #[inline]
     pub fn write_i8(&mut self, n: i8) {
         self.bytes.push(n as u8);
     }
 
     /// Writes a 16-bit unsigned integer to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_u16_le(&mut self, n: u16) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 16-bit unsigned integer to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_u16_be(&mut self, n: u16) {
         self.write(&n.to_be_bytes());
     }
 
     /// Writes a 16-bit signed integer to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_i16_le(&mut self, n: i16) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 16-bit signed integer to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_i16_be(&mut self, n: i16) {
         self.write(&n.to_be_bytes());
     }
 
     /// Writes a 32-bit unsigned integer to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_u32_le(&mut self, n: u32) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 32-bit unsigned integer to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_u32_be(&mut self, n: u32) {
         self.write(&n.to_be_bytes());
     }
 
     /// Writes a 32-bit signed integer to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_i32_le(&mut self, n: i32) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 32-bit signed integer to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_i32_be(&mut self, n: i32) {
         self.write(&n.to_be_bytes());
     }
 
     /// Writes a 64-bit unsigned integer to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_u64_le(&mut self, n: u64) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 64-bit unsigned integer to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_u64_be(&mut self, n: u64) {
         self.write(&n.to_be_bytes());
     }
 
     /// Writes a 64-bit signed integer to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_i64_le(&mut self, n: i64) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 64-bit signed integer to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_i64_be(&mut self, n: i64) {
         self.write(&n.to_be_bytes());
     }
 
     /// Writes a 32-bit floating-point number to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_f32_le(&mut self, n: f32) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 32-bit floating-point number to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_f32_be(&mut self, n: f32) {
         self.write(&n.to_be_bytes());
     }
 
     /// Writes a 64-bit floating-point number to the `SketchBytes` in little-endian byte order.
+    #[inline]
     pub fn write_f64_le(&mut self, n: f64) {
         self.write(&n.to_le_bytes());
     }
 
     /// Writes a 64-bit floating-point number to the `SketchBytes` in big-endian byte order.
+    #[inline]
     pub fn write_f64_be(&mut self, n: f64) {
         self.write(&n.to_be_bytes());
     }

datasketches/src/error.rs

@@ -100,10 +100,14 @@ impl Error {
         Self::new(ErrorKind::InvalidData, msg)
     }
 
+    #[cold]
+    #[inline(never)]
     pub(crate) fn insufficient_data(msg: impl fmt::Display) -> Self {
         Self::deserial(format!("insufficient data: {msg}"))
     }
 
+    #[cold]
+    #[inline(never)]
     pub(crate) fn insufficient_data_of(context: &'static str, msg: impl fmt::Display) -> Self {
         Self::deserial(format!("insufficient data ({context}): {msg}"))
     }

datasketches/src/thetafamily/common/hash_table.rs

@@ -169,6 +169,7 @@ where
     }
 
     /// Hash a value with the table seed and return the hash.
+    #[inline]
     pub fn hash<T: Hash>(&self, value: T) -> u64 {
         let mut hasher = MurmurHash3X64128::with_seed(self.seed);
         value.hash(&mut hasher);

datasketches/src/thetafamily/theta/hash_table.rs

@@ -37,18 +37,21 @@ pub struct ThetaEntry {
 }
 
 impl ThetaEntry {
+    #[inline]
     pub(super) fn new(hash: u64) -> Self {
         let hash = NonZeroU64::new(hash).expect("hash must be non-zero");
         Self { hash }
     }
 
     /// Returns the hash used as this entry's key.
+    #[inline]
     pub fn hash(&self) -> u64 {
         self.hash.get()
     }
 }
 
 impl SketchEntry for ThetaEntry {
+    #[inline]
     fn hash(&self) -> u64 {
         self.hash.get()
     }

datasketches/src/thetafamily/tuple/serialization.rs

@@ -68,14 +68,17 @@ pub trait TupleSummaryValue: Sized {
 macro_rules! impl_primitive_summary {
     ($name:ty, $read:ident, $write:ident) => {
         impl TupleSummaryValue for $name {
+            #[inline]
             fn serialize_size(&self) -> usize {
                 size_of::<$name>()
             }
 
+            #[inline]
             fn serialize_value(&self, bytes: &mut SketchBytes) {
                 bytes.$write(*self);
             }
 
+            #[inline]
             fn deserialize_value(cursor: &mut SketchSlice<'_>) -> Result<Self, Error> {
                 cursor
                     .$read()