diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ebe1ed..9b8c76d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,8 @@ jobs: - "" - "--no-default-features --features deflate" - "--no-default-features --features zstd" + - "--no-default-features --features zstd-rust" + - "--features zstd-rust" steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -52,7 +54,7 @@ jobs: default: true - name: assert deps don't exist with --no-default-features run: | - for DEP in libflate zstd; do + for DEP in libflate zstd structured-zstd; do if cargo tree --no-default-features --invert $DEP; then echo "Dependency $DEP should not exist with --no-default-features" exit 1 @@ -70,3 +72,11 @@ jobs: echo "Feature `zstd` should not create dep on `libflate`" exit 1 fi + - name: assert `zstd-rust` feature doesn't create dep on the C zstd bindings + run: | + for DEP in zstd zstd-sys libflate; do + if cargo tree --no-default-features --features zstd-rust --invert $DEP; then + echo "Feature `zstd-rust` should not create dep on $DEP" + exit 1 + fi + done diff --git a/Cargo.toml b/Cargo.toml index 81fe4fc..a313b30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ license = "Apache-2.0" [workspace.dependencies] libflate = "2.3.0" zstd = "0.14.0" +structured-zstd = "0.0.59" [package] name = "include-flate" @@ -34,4 +35,7 @@ include-flate-compress = { version = "0.3.4", path = "compress", default-feature default = ["deflate", "zstd"] deflate = ["include-flate-codegen/deflate", "include-flate-compress/deflate"] zstd = ["include-flate-codegen/zstd", "include-flate-compress/zstd"] +# Zstandard through the pure-Rust `structured-zstd`, with no C toolchain, +# selected by `with zstd_rust`. Frames are interchangeable with `with zstd`. +zstd-rust = ["include-flate-codegen/zstd-rust", "include-flate-compress/zstd-rust"] no-compression-warnings = ["include-flate-codegen/no-compression-warnings"] diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 0f0f4b3..889a2ce 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -22,4 +22,5 @@ proc-macro-error3 = "3.0.2" default = ["deflate", "zstd"] deflate = ["include-flate-compress/deflate"] zstd = ["include-flate-compress/zstd"] +zstd-rust = ["include-flate-compress/zstd-rust"] no-compression-warnings = [] diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index bfd379d..710ff69 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -72,6 +72,7 @@ pub fn deflate_utf8_file(ts: TokenStream) -> TokenStream { /// flate!(pub static DATA: [u8] from "assets/009f.dat"); // default, DEFLATE /// flate!(pub static DATA: [u8] from "assets/009f.dat" with zstd); // Use Zstd for this file spcifically /// flate!(pub static DATA: [u8] from "assets/009f.dat" with deflate); // Explicitly use DEFLATE. +/// flate!(pub static DATA: [u8] from "assets/009f.dat" with zstd_rust); // Zstd without the C library /// ``` struct FlateArgs { path: syn::LitStr, @@ -105,6 +106,17 @@ impl syn::parse::Parse for FlateArgs { } #[cfg(not(feature = "zstd"))] return Err(Error::new(input.span(), "Please enable the `zstd` feature")); + } else if lookahead.peek(kw::zstd_rust) { + #[cfg(feature = "zstd-rust")] + { + input.parse::()?; + Some(CompressionMethodTy(CompressionMethod::ZstdRust)) + } + #[cfg(not(feature = "zstd-rust"))] + return Err(Error::new( + input.span(), + "Please enable the `zstd-rust` feature", + )); } else { return Err(lookahead.error()); } @@ -117,6 +129,7 @@ impl syn::parse::Parse for FlateArgs { mod kw { syn::custom_keyword!(deflate); syn::custom_keyword!(zstd); + syn::custom_keyword!(zstd_rust); } #[derive(Debug, Default)] diff --git a/compress/Cargo.toml b/compress/Cargo.toml index 2e6a4f5..cdd2277 100644 --- a/compress/Cargo.toml +++ b/compress/Cargo.toml @@ -11,8 +11,12 @@ description = "Compression algorithm provider" [dependencies] libflate = { workspace = true, optional = true } zstd = { workspace = true, optional = true } +structured-zstd = { workspace = true, optional = true } [features] default = ["deflate", "zstd"] deflate = ["dep:libflate"] zstd = ["dep:zstd"] +# Zstandard through the pure-Rust `structured-zstd`, as `CompressionMethod::ZstdRust`. +# Same frame format as `CompressionMethod::Zstd`. +zstd-rust = ["dep:structured-zstd"] diff --git a/compress/src/lib.rs b/compress/src/lib.rs index e6b0a47..ae9e52c 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -13,8 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(not(any(feature = "zstd", feature = "deflate")))] -compile_error!("You must enable either the `deflate` or `zstd` feature."); +#[cfg(not(any(feature = "zstd", feature = "zstd-rust", feature = "deflate")))] +compile_error!("You must enable either the `deflate`, `zstd` or `zstd-rust` feature."); use std::{ fmt, @@ -30,12 +30,104 @@ use zstd::Decoder as ZstdDecoder; #[cfg(feature = "zstd")] use zstd::Encoder as ZstdEncoder; +// The pure-Rust decoder keeps its state inline (the C one holds a pointer to +// it), so it is boxed to keep the enum below the size of its other variants. +#[cfg(feature = "zstd-rust")] +type ZstdRustDecoder = + Box>; + +/// Decodes a Zstandard stream held in memory: every frame, skippable frames +/// skipped (RFC 8878 3). +#[cfg(feature = "zstd-rust")] +fn decode_zstd_rust(bytes: &[u8]) -> io::Result> { + use structured_zstd::decoding::errors::ReadFrameHeaderError; + use structured_zstd::decoding::{ + FrameContentSize, FrameDecoder, StreamingDecoder, find_frame_compressed_size, + read_frame_content_size, + }; + + fn invalid(err: impl fmt::Debug) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, format!("{err:?}")) + } + + // RFC 8878 3: compressed data is one or more frames. + if bytes.is_empty() { + return Err(invalid("empty input holds no Zstandard frame")); + } + + // Walk the frames first: when every one declares Frame_Content_Size + // (optional, RFC 8878 3.1.1.1.2), the stream decodes in one call straight + // into a buffer of exactly their total size. + let mut total = 0u64; + let mut all_declared = true; + let mut offset = 0; + while offset < bytes.len() { + let frame = &bytes[offset..]; + let len = find_frame_compressed_size(frame).map_err(invalid)?; + match read_frame_content_size(frame) { + Ok(size) => { + if let FrameContentSize::Known(size) = size { + // RFC 8878 3.1.1.2: every block takes at least its 3-byte + // header on disk and regenerates at most 128 KiB, so a + // frame declaring more than that cannot be honest. + let max = (len as u64 / 3) * (128 * 1024); + if size > max { + return Err(invalid(format!( + "frame declares {size} bytes but its {len} bytes hold at most {max}" + ))); + } + // Each term is bounded by its frame's length as above, so + // the sum stays far below u64::MAX. + total += size; + } else { + all_declared = false; + } + } + Err(ReadFrameHeaderError::SkipFrame { .. }) => {} + Err(err) => return Err(invalid(err)), + } + offset += len; + } + + if all_declared { + let size = usize::try_from(total).map_err(invalid)?; + let mut out = vec![0; size]; + let written = FrameDecoder::new() + .decode_all(bytes, &mut out) + .map_err(invalid)?; + if written != size { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("frames declare {size} bytes but hold {written}"), + )); + } + return Ok(out); + } + + let mut out = Vec::new(); + StreamingDecoder::new(bytes) + .map_err(invalid)? + .read_to_end(&mut out)?; + Ok(out) +} + +/// Collects the whole input and encodes it as one frame on finish, so the +/// frame declares `Frame_Content_Size` and `decompress_slice` can decode it +/// into an exactly sized buffer in one call. +#[cfg(feature = "zstd-rust")] +pub struct ZstdRustEncoder { + input: Vec, + sink: W, +} + #[derive(Debug)] pub enum CompressionError { #[cfg(feature = "deflate")] DeflateError(io::Error), #[cfg(feature = "zstd")] ZstdError(io::Error), + #[cfg(feature = "zstd-rust")] + ZstdRustError(io::Error), IoError(io::Error), } @@ -52,17 +144,24 @@ impl fmt::Display for CompressionError { CompressionError::DeflateError(err) => write!(f, "Deflate error: {}", err), #[cfg(feature = "zstd")] CompressionError::ZstdError(err) => write!(f, "Zstd error: {}", err), + #[cfg(feature = "zstd-rust")] + CompressionError::ZstdRustError(err) => write!(f, "Zstd (pure Rust) error: {}", err), CompressionError::IoError(err) => write!(f, "I/O error: {}", err), } } } +/// `Zstd` and `ZstdRust` produce and read the same Zstandard frame format: +/// `Zstd` through the C library, `ZstdRust` through the pure-Rust +/// `structured-zstd`, which needs no C toolchain. #[derive(Debug, Copy, Clone, PartialEq)] pub enum CompressionMethod { #[cfg(feature = "deflate")] Deflate, #[cfg(feature = "zstd")] Zstd, + #[cfg(feature = "zstd-rust")] + ZstdRust, } impl CompressionMethod { @@ -79,7 +178,7 @@ impl CompressionMethod { clippy::derivable_impls, reason = "cfg_attr on defaults could be confusing" )] -#[cfg(any(feature = "deflate", feature = "zstd"))] +#[cfg(any(feature = "deflate", feature = "zstd", feature = "zstd-rust"))] impl Default for CompressionMethod { fn default() -> Self { #[cfg(feature = "deflate")] @@ -90,6 +189,10 @@ impl Default for CompressionMethod { { Self::Zstd } + #[cfg(all(not(feature = "deflate"), not(feature = "zstd"), feature = "zstd-rust"))] + { + Self::ZstdRust + } } } @@ -100,6 +203,8 @@ impl fmt::Display for CompressionMethod { Self::Deflate => "deflate", #[cfg(feature = "zstd")] Self::Zstd => "zstd", + #[cfg(feature = "zstd-rust")] + Self::ZstdRust => "zstd_rust", }) } } @@ -109,6 +214,8 @@ pub enum FlateEncoder { Deflate(DeflateEncoder), #[cfg(feature = "zstd")] Zstd(ZstdEncoder<'static, W>), + #[cfg(feature = "zstd-rust")] + ZstdRust(ZstdRustEncoder), } impl FlateEncoder { @@ -120,6 +227,11 @@ impl FlateEncoder { CompressionMethod::Zstd => ZstdEncoder::new(write, 0) .map(FlateEncoder::Zstd) .map_err(CompressionError::ZstdError), + #[cfg(feature = "zstd-rust")] + CompressionMethod::ZstdRust => Ok(FlateEncoder::ZstdRust(ZstdRustEncoder { + input: Vec::new(), + sink: write, + })), } } } @@ -131,6 +243,11 @@ impl Write for FlateEncoder { FlateEncoder::Deflate(encoder) => encoder.write(buf), #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.write(buf), + #[cfg(feature = "zstd-rust")] + FlateEncoder::ZstdRust(encoder) => { + encoder.input.extend_from_slice(buf); + Ok(buf.len()) + } } } @@ -140,12 +257,17 @@ impl Write for FlateEncoder { FlateEncoder::Deflate(encoder) => encoder.flush(), #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.flush(), + // Nothing is encoded before the frame is finished. + #[cfg(feature = "zstd-rust")] + FlateEncoder::ZstdRust(_) => Ok(()), } } } impl FlateEncoder { - fn finish_encode(self) -> Result { + /// Ends the stream and returns the writer. Every method needs this to + /// write its final bytes; `ZstdRust` writes the whole frame here. + pub fn finish(self) -> Result { match self { #[cfg(feature = "deflate")] FlateEncoder::Deflate(encoder) => encoder @@ -154,15 +276,29 @@ impl FlateEncoder { .map_err(CompressionError::DeflateError), #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.finish().map_err(CompressionError::ZstdError), + // `Default` is this backend's counterpart of zstd level 3, which + // the C backend selects through level 0. + #[cfg(feature = "zstd-rust")] + FlateEncoder::ZstdRust(ZstdRustEncoder { input, mut sink }) => { + let frame = structured_zstd::encoding::compress_slice_to_vec( + &input, + structured_zstd::encoding::CompressionLevel::Default, + ); + sink.write_all(&frame) + .map_err(CompressionError::ZstdRustError)?; + Ok(sink) + } } } } -pub enum FlateDecoder { +pub enum FlateDecoder { #[cfg(feature = "deflate")] Deflate(DeflateDecoder), #[cfg(feature = "zstd")] Zstd(ZstdDecoder<'static, std::io::BufReader>), + #[cfg(feature = "zstd-rust")] + ZstdRust(ZstdRustDecoder), } impl FlateDecoder { @@ -175,6 +311,10 @@ impl FlateDecoder { let decoder = ZstdDecoder::new(read)?; Ok(FlateDecoder::Zstd(decoder)) } + #[cfg(feature = "zstd-rust")] + CompressionMethod::ZstdRust => structured_zstd::decoding::StreamingDecoder::new(read) + .map(|decoder| FlateDecoder::ZstdRust(Box::new(decoder))) + .map_err(|err| CompressionError::ZstdRustError(io::Error::other(err))), } } } @@ -186,6 +326,8 @@ impl Read for FlateDecoder { FlateDecoder::Deflate(decoder) => decoder.read(buf), #[cfg(feature = "zstd")] FlateDecoder::Zstd(decoder) => decoder.read(buf), + #[cfg(feature = "zstd-rust")] + FlateDecoder::ZstdRust(decoder) => decoder.read(buf), } } } @@ -201,7 +343,7 @@ where { let mut encoder = method.encoder(writer)?; io::copy(reader, &mut encoder)?; - encoder.finish_encode().map(|_| ()) + encoder.finish().map(|_| ()) } pub fn apply_decompression( @@ -213,3 +355,27 @@ pub fn apply_decompression( io::copy(&mut decoder, &mut writer)?; Ok(()) } + +/// Decodes a whole compressed buffer held in memory. +/// +/// Meant for data this crate encoded, such as what `flate!` embeds. For +/// `ZstdRust` the output is allocated up front at the size the frames declare, +/// bounded only by what their length could hold, so a crafted input can ask +/// for far more memory than it decodes to; decode untrusted input through +/// [`CompressionMethod::decoder`] instead. +pub fn decompress_slice( + bytes: &[u8], + method: CompressionMethod, +) -> Result, CompressionError> { + #[cfg(feature = "zstd-rust")] + if method == CompressionMethod::ZstdRust { + return decode_zstd_rust(bytes).map_err(CompressionError::ZstdRustError); + } + + let mut out = Vec::new(); + apply_decompression(bytes, &mut out, method)?; + Ok(out) +} + +#[cfg(test)] +mod tests; diff --git a/compress/src/tests.rs b/compress/src/tests.rs new file mode 100644 index 0000000..2e6aaa6 --- /dev/null +++ b/compress/src/tests.rs @@ -0,0 +1,351 @@ +// include-flate +// Copyright (C) SOFe +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(feature = "zstd-rust")] + +use super::{CompressionMethod, apply_compression, decompress_slice}; + +fn sample() -> Vec { + (0..200_000u32) + .flat_map(|i| format!("{} {} ", i % 977, i % 13).into_bytes()) + .collect() +} + +fn encode(data: &[u8], method: CompressionMethod) -> Vec { + let mut frame = Vec::new(); + apply_compression(&mut &data[..], &mut frame, method).unwrap(); + frame +} + +/// `Zstd` and `ZstdRust` share the Zstandard frame format, so each one +/// decodes what the other encoded, through both decoding entry points. +#[cfg(feature = "zstd")] +fn round_trip(encode_with: CompressionMethod, decode_with: CompressionMethod) { + use super::apply_decompression; + + let data = sample(); + let frame = encode(&data, encode_with); + + let mut decoded = Vec::new(); + apply_decompression(&frame[..], &mut decoded, decode_with).unwrap(); + assert_eq!(decoded, data); + + assert_eq!(decompress_slice(&frame, decode_with).unwrap(), data); +} + +#[cfg(feature = "zstd")] +#[test] +fn a_zstd_frame_decodes_with_zstd_rust() { + round_trip(CompressionMethod::Zstd, CompressionMethod::ZstdRust); +} + +#[cfg(feature = "zstd")] +#[test] +fn a_zstd_rust_frame_decodes_with_zstd() { + round_trip(CompressionMethod::ZstdRust, CompressionMethod::Zstd); +} + +/// The one-shot decode sizes its output from `Frame_Content_Size`, so a +/// `ZstdRust` frame must declare it. +#[test] +fn a_zstd_rust_frame_declares_its_content_size() { + use structured_zstd::decoding::{FrameContentSize, read_frame_content_size}; + + let data = sample(); + let frame = encode(&data, CompressionMethod::ZstdRust); + assert_eq!( + read_frame_content_size(&frame).unwrap(), + FrameContentSize::Known(data.len() as u64) + ); + assert_eq!( + decompress_slice(&frame, CompressionMethod::ZstdRust).unwrap(), + data + ); +} + +/// Empty input still makes a frame that declares its (zero) size and decodes +/// back to nothing. +#[test] +fn an_empty_zstd_rust_frame_round_trips() { + let frame = encode(&[], CompressionMethod::ZstdRust); + assert!( + decompress_slice(&frame, CompressionMethod::ZstdRust) + .unwrap() + .is_empty() + ); +} + +/// `Frame_Content_Size` is optional (RFC 8878 3.1.1.1.2); a frame without it +/// still decodes, through the streaming decoder. +#[test] +fn a_zstd_rust_frame_without_content_size_decodes() { + use structured_zstd::decoding::{FrameContentSize, read_frame_content_size}; + + let data = sample(); + let frame = frame_without_content_size(&data); + assert_eq!( + read_frame_content_size(&frame).unwrap(), + FrameContentSize::Unknown + ); + + assert_eq!( + decompress_slice(&frame, CompressionMethod::ZstdRust).unwrap(), + data + ); +} + +/// A frame whose blocks hold less than its header declares is reported as an +/// error, not returned with a zero-filled tail. +#[test] +fn a_zstd_rust_frame_shorter_than_declared_is_an_error() { + use structured_zstd::decoding::{FrameContentSize, read_frame_content_size}; + + let data = sample(); + let mut frame = encode(&data, CompressionMethod::ZstdRust); + + // RFC 8878 3.1.1.1: magic (4), descriptor (1), then the window descriptor + // unless Single_Segment_flag is set, the dictionary ID, and the + // little-endian Frame_Content_Size field. + let descriptor = frame[4]; + let single_segment = descriptor & 0x20 != 0; + let dict_id_len = [0, 1, 2, 4][usize::from(descriptor & 0x03)]; + let fcs_len = match descriptor >> 6 { + 0 => usize::from(single_segment), + 1 => 2, + 2 => 4, + _ => 8, + }; + assert_eq!(fcs_len, 4, "the sample needs a 4-byte FCS field"); + let at = 5 + usize::from(!single_segment) + dict_id_len; + let declared = u32::from_le_bytes(frame[at..at + 4].try_into().unwrap()); + frame[at..at + 4].copy_from_slice(&(declared + 1).to_le_bytes()); + assert_eq!( + read_frame_content_size(&frame).unwrap(), + FrameContentSize::Known(data.len() as u64 + 1) + ); + + assert!(decompress_slice(&frame, CompressionMethod::ZstdRust).is_err()); +} + +/// A frame encoded without `Frame_Content_Size`, as a streaming encoder that +/// does not know the input length writes it. +fn frame_without_content_size(data: &[u8]) -> Vec { + use std::io::Write; + use structured_zstd::encoding::{CompressionLevel, StreamingEncoder}; + + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Default); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + +/// RFC 8878 3.1.2: a skippable frame carrying `payload`. +fn skippable_frame(payload: &[u8]) -> Vec { + let mut frame = 0x184D_2A50u32.to_le_bytes().to_vec(); + frame.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + frame.extend_from_slice(payload); + frame +} + +fn decode_streaming(bytes: &[u8]) -> Vec { + let mut out = Vec::new(); + super::apply_decompression(bytes, &mut out, CompressionMethod::ZstdRust).unwrap(); + out +} + +/// The encoder handed out by `CompressionMethod::encoder` can be finished +/// by its caller, which is what writes the frame. +#[test] +fn the_public_zstd_rust_encoder_can_be_finished() { + use std::io::Write; + + let data = sample(); + let mut encoder = CompressionMethod::ZstdRust.encoder(Vec::new()).unwrap(); + encoder.write_all(&data).unwrap(); + let frame = encoder.finish().unwrap(); + assert_eq!( + decompress_slice(&frame, CompressionMethod::ZstdRust).unwrap(), + data + ); +} + +/// A tiny frame declaring a content size no frame of its length can hold is +/// an error, not a capacity-overflow panic or an allocation of that size. +#[test] +fn a_zstd_rust_frame_declaring_an_impossible_size_is_an_error() { + // RFC 8878 3.1.1: magic, descriptor with an 8-byte Frame_Content_Size and + // Single_Segment_flag, the size, then one empty last raw block. + let mut frame = 0xFD2F_B528u32.to_le_bytes().to_vec(); + frame.push(0xE0); + frame.extend_from_slice(&u64::MAX.to_le_bytes()); + frame.extend_from_slice(&[0x01, 0x00, 0x00]); + + assert!(decompress_slice(&frame, CompressionMethod::ZstdRust).is_err()); +} + +/// RFC 8878 3: a stream is a sequence of frames. Frames that declare their +/// size decode in one call, through every frame, on both entry points. +#[test] +fn concatenated_zstd_rust_frames_decode_completely() { + let data = sample(); + let (first, second) = data.split_at(data.len() / 3); + let mut stream = encode(first, CompressionMethod::ZstdRust); + stream.extend_from_slice(&encode(second, CompressionMethod::ZstdRust)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// Frames without a declared size, with a skippable frame between them, +/// decode completely; the skippable frame contributes nothing. +#[test] +fn concatenated_frames_without_content_size_decode_completely() { + let data = sample(); + let (first, second) = data.split_at(data.len() / 3); + let mut stream = frame_without_content_size(first); + stream.extend_from_slice(&skippable_frame(b"meta")); + stream.extend_from_slice(&frame_without_content_size(second)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// One frame with a declared size and one without still decode completely. +#[test] +fn frames_with_and_without_content_size_decode_completely() { + let data = sample(); + let (first, second) = data.split_at(data.len() / 2); + let mut stream = encode(first, CompressionMethod::ZstdRust); + stream.extend_from_slice(&frame_without_content_size(second)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// RFC 8878 3: compressed data is one or more frames, so an empty input is +/// not a stream (a fully truncated frame) and must not decode to nothing. +#[test] +fn empty_input_is_not_a_zstd_rust_stream() { + assert!(decompress_slice(&[], CompressionMethod::ZstdRust).is_err()); + let mut out = Vec::new(); + assert!(super::apply_decompression(&[][..], &mut out, CompressionMethod::ZstdRust).is_err()); +} + +/// A stream of skippable frames only holds no data: both entry points decode +/// it to nothing. +#[test] +fn skippable_frames_alone_decode_to_nothing() { + let mut stream = skippable_frame(b"one"); + stream.extend_from_slice(&skippable_frame(b"two")); + + assert!( + decompress_slice(&stream, CompressionMethod::ZstdRust) + .unwrap() + .is_empty() + ); + assert!(decode_streaming(&stream).is_empty()); +} + +/// The `Read` decoder hands concatenated frames out through reads smaller +/// than a block, across the frame boundary. +#[test] +fn the_zstd_rust_decoder_reads_concatenated_frames_in_small_pieces() { + use std::io::Read; + + let data = sample(); + let (first, second) = data.split_at(data.len() / 3); + let mut stream = encode(first, CompressionMethod::ZstdRust); + stream.extend_from_slice(&skippable_frame(b"meta")); + stream.extend_from_slice(&frame_without_content_size(second)); + + let mut decoder = CompressionMethod::ZstdRust.decoder(&stream[..]).unwrap(); + let mut out = Vec::new(); + let mut piece = [0u8; 7]; + loop { + let n = decoder.read(&mut piece).unwrap(); + if n == 0 { + break; + } + out.extend_from_slice(&piece[..n]); + } + assert_eq!(out, data); +} + +/// The `Read` decoder stays streaming: once a whole frame has arrived from a +/// source that stays open (here it answers `WouldBlock` after the frame), its +/// bytes are handed out without waiting for the source to end. +#[test] +fn the_zstd_rust_decoder_does_not_wait_for_the_source_to_end() { + use std::io::{self, Read}; + + struct OpenSource<'a>(&'a [u8]); + impl Read for OpenSource<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.0.is_empty() { + return Err(io::ErrorKind::WouldBlock.into()); + } + let n = buf.len().min(self.0.len()); + buf[..n].copy_from_slice(&self.0[..n]); + self.0 = &self.0[n..]; + Ok(n) + } + } + + let data = sample(); + let frame = encode(&data, CompressionMethod::ZstdRust); + let mut decoder = CompressionMethod::ZstdRust + .decoder(OpenSource(&frame)) + .unwrap(); + let mut out = vec![0; data.len()]; + decoder.read_exact(&mut out).unwrap(); + assert_eq!(out, data); +} + +/// A skippable frame ahead of the data is skipped by both entry points. +#[test] +fn a_leading_skippable_frame_is_skipped() { + let data = sample(); + let mut stream = skippable_frame(&[7; 16]); + stream.extend_from_slice(&encode(&data, CompressionMethod::ZstdRust)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// Bytes after the last frame that do not form a frame are an error, not +/// silently dropped. +#[test] +fn trailing_garbage_after_a_zstd_rust_frame_is_an_error() { + let data = sample(); + let mut stream = encode(&data, CompressionMethod::ZstdRust); + stream.extend_from_slice(b"garbage"); + + assert!(decompress_slice(&stream, CompressionMethod::ZstdRust).is_err()); + let mut out = Vec::new(); + assert!( + super::apply_decompression(&stream[..], &mut out, CompressionMethod::ZstdRust).is_err() + ); +} diff --git a/src/lib.rs b/src/lib.rs index 764310b..a4d0b77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,7 @@ /// The low-level macros used by this crate. pub use include_flate_codegen as codegen; -use include_flate_compress::apply_decompression; +use include_flate_compress::decompress_slice; use std::string::FromUtf8Error; pub use include_flate_compress::CompressionMethod; @@ -61,7 +61,11 @@ pub use include_flate_compress::CompressionMethod; /// - If `$type` is `str` but the file is not fully valid UTF-8 /// /// # Algorithm -/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5]. +/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5] by default. +/// `with zstd` selects Zstandard through the C library (the `zstd` feature), and +/// `with zstd_rust` selects Zstandard through the pure-Rust [`structured-zstd`][7] +/// (the `zstd-rust` feature), which needs no C toolchain. Both produce and read the same +/// frame format. /// /// # Examples /// Below are some basic examples. For actual compiled examples, see the [`tests`][6] directory. @@ -88,6 +92,7 @@ pub use include_flate_compress::CompressionMethod; /// [4]: https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates /// [5]: https://docs.rs/libflate/0.1.26/libflate/ /// [6]: https://github.com/SOF3/include-flate/tree/master/tests +/// [7]: https://docs.rs/structured-zstd #[macro_export] macro_rules! flate { ($(#[$meta:meta])* @@ -139,6 +144,9 @@ macro_rules! __parse_algo { (zstd) => { $crate::CompressionMethod::Zstd }; + (zstd_rust) => { + $crate::CompressionMethod::ZstdRust + }; ($other:ident) => { compile_error!("Unknown compression algorithm: {}", stringify!($other)) }; @@ -176,18 +184,11 @@ impl IFlate { #[doc(hidden)] #[allow(private_interfaces)] pub fn decode(bytes: &[u8], algo: Option) -> Vec { - use std::io::Cursor; - let algo = algo.unwrap_or_default(); - let mut source = Cursor::new(bytes); - let mut ret = Vec::new(); - - match apply_decompression(&mut source, &mut ret, algo) { - Ok(_) => {} + match decompress_slice(bytes, algo) { + Ok(ret) => ret, Err(err) => panic!("Compiled `{:?}` buffer was corrupted: {:?}", algo, err), } - - ret } #[doc(hidden)] diff --git a/test_util.rs b/test_util.rs index eb42d06..0ea1d27 100644 --- a/test_util.rs +++ b/test_util.rs @@ -18,12 +18,14 @@ use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::str::from_utf8; +#[cfg(all(feature = "zstd", feature = "zstd-rust"))] +use include_flate_compress::decompress_slice; use include_flate_compress::{CompressionMethod, apply_compression, apply_decompression}; pub fn get_file_path>(relative_from: Option<&Path>, path: P) -> PathBuf { let cargo_manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); let default_base_path = Path::new(&cargo_manifest_dir); - let base_path = relative_from.unwrap_or_else(|| default_base_path); + let base_path = relative_from.unwrap_or(default_base_path); base_path.join("assets").join(path) } @@ -57,11 +59,30 @@ pub fn verify_compression>(name: P, data: &[u8], method: Compress assert_ne!(compressed_buffer.as_slice(), decompressed_buffer.as_slice()); } +/// Encodes the file with `encode` and checks that `decode` restores it exactly. +#[cfg(all(feature = "zstd", feature = "zstd-rust"))] +pub fn verify_cross>(name: P, encode: CompressionMethod, decode: CompressionMethod) { + let original = read_file(&name); + let mut compressed = Vec::new(); + apply_compression(&mut &original[..], &mut compressed, encode).unwrap(); + let mut decompressed = Vec::new(); + apply_decompression(&compressed[..], &mut decompressed, decode).unwrap(); + assert_eq!(decompressed, original); + assert_eq!(decompress_slice(&compressed, decode).unwrap(), original); +} + pub fn verify>(name: P, data: &[u8]) { #[cfg(feature = "deflate")] verify_compression(&name, data, CompressionMethod::Deflate); #[cfg(feature = "zstd")] verify_compression(&name, data, CompressionMethod::Zstd); + #[cfg(feature = "zstd-rust")] + verify_compression(&name, data, CompressionMethod::ZstdRust); + #[cfg(all(feature = "zstd", feature = "zstd-rust"))] + { + verify_cross(&name, CompressionMethod::Zstd, CompressionMethod::ZstdRust); + verify_cross(&name, CompressionMethod::ZstdRust, CompressionMethod::Zstd); + } assert_eq!(read_file(&name), data); } diff --git a/tests/009f.rs b/tests/009f.rs index 72cc022..188cb3d 100644 --- a/tests/009f.rs +++ b/tests/009f.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: [u8] from "assets/009f.dat" with zstd); flate!(pub static DATA4: IFlate from "assets/009f.dat" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/009f.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/009f.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/009f.dat" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("009f.dat", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("009f.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("009f.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("009f.dat", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/ascii-control.rs b/tests/ascii-control.rs index bbf9b0e..a1bd075 100644 --- a/tests/ascii-control.rs +++ b/tests/ascii-control.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: str from "assets/ascii-control.txt" with zstd); flate!(pub static DATA4: IFlate from "assets/ascii-control.txt" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/ascii-control.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/ascii-control.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/ascii-control.txt" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("ascii-control.txt", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("ascii-control.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("ascii-control.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("ascii-control.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/ascii-printable.rs b/tests/ascii-printable.rs index db46312..f315b8c 100644 --- a/tests/ascii-printable.rs +++ b/tests/ascii-printable.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: str from "assets/ascii-printable.txt" with zstd); flate!(pub static DATA4: IFlate from "assets/ascii-printable.txt" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/ascii-printable.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/ascii-printable.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/ascii-printable.txt" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("ascii-printable.txt", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("ascii-printable.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("ascii-printable.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("ascii-printable.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/base64.rs b/tests/base64.rs index 5171c3f..acf3fc8 100644 --- a/tests/base64.rs +++ b/tests/base64.rs @@ -29,6 +29,10 @@ flate!(pub static DATA3: str from "assets/base64.txt" with zstd); flate!(pub static DATA4: IFlate from "assets/base64.txt" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/base64.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/base64.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/base64.txt" with zstd_rust); #[test] fn test() { @@ -41,4 +45,8 @@ fn test() { verify_iflate("base64.txt", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("base64.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("base64.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("base64.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/chinese.rs b/tests/chinese.rs index a092f05..6e292a3 100644 --- a/tests/chinese.rs +++ b/tests/chinese.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: str from "assets/chinese.txt" with zstd); flate!(pub static DATA4: IFlate from "assets/chinese.txt" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/chinese.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/chinese.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/chinese.txt" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("chinese.txt", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("chinese.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("chinese.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("chinese.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/emoji.rs b/tests/emoji.rs index 24efc71..835c625 100644 --- a/tests/emoji.rs +++ b/tests/emoji.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: str from "assets/emoji.txt" with zstd); flate!(pub static DATA4: IFlate from "assets/emoji.txt" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/emoji.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/emoji.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/emoji.txt" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("emoji.txt", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("emoji.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("emoji.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("emoji.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/ff.rs b/tests/ff.rs index 8bc13e1..787e830 100644 --- a/tests/ff.rs +++ b/tests/ff.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: [u8] from "assets/ff.dat" with zstd); flate!(pub static DATA4: IFlate from "assets/ff.dat" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/ff.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/ff.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/ff.dat" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("ff.dat", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("ff.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("ff.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("ff.dat", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/random.rs b/tests/random.rs index e1254e0..05ac978 100644 --- a/tests/random.rs +++ b/tests/random.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: [u8] from "assets/random.dat" with zstd); flate!(pub static DATA4: IFlate from "assets/random.dat" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/random.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/random.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/random.dat" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("random.dat", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("random.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("random.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("random.dat", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/zero.rs b/tests/zero.rs index add2ea9..68239c0 100644 --- a/tests/zero.rs +++ b/tests/zero.rs @@ -26,6 +26,10 @@ flate!(pub static DATA3: [u8] from "assets/zero.dat" with zstd); flate!(pub static DATA4: IFlate from "assets/zero.dat" with deflate); #[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/zero.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/zero.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/zero.dat" with zstd_rust); #[test] fn test() { @@ -38,4 +42,8 @@ fn test() { verify_iflate("zero.dat", CompressionMethod::Deflate, &DATA4); #[cfg(feature = "zstd")] verify_iflate("zero.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("zero.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("zero.dat", CompressionMethod::ZstdRust, &DATA7); }