From e8713e0d2ee2a6b98dc2d581097706edff83e306 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 13:00:13 +0300 Subject: [PATCH 1/9] Add a pure-Rust Zstandard backend behind the zstd-rust feature The zstd feature binds the C library through zstd-sys, which needs a C toolchain at build time and links C code into every consumer. The new zstd-rust feature provides the same CompressionMethod::Zstd and the same `with zstd` syntax through structured-zstd, a pure-Rust implementation of the format. Frames are interchangeable between the two backends; when both features are enabled, zstd-rust is used. - compress: backend selected by feature, same public API. - codegen, root crate: feature passed through. - Tests run under zstd-rust as well; a new interop test decodes C-zstd frames with the Rust backend and the reverse. - CI: test matrix entries for zstd-rust, and a check that it pulls in neither zstd, zstd-sys nor libflate. --- .github/workflows/ci.yml | 12 ++++- Cargo.toml | 4 ++ codegen/Cargo.toml | 1 + codegen/src/lib.rs | 9 ++-- compress/Cargo.toml | 4 ++ compress/src/lib.rs | 83 ++++++++++++++++++++++++---------- compress/tests/zstd_interop.rs | 53 ++++++++++++++++++++++ src/lib.rs | 7 ++- test_util.rs | 2 +- tests/009f.rs | 8 ++-- tests/ascii-control.rs | 8 ++-- tests/ascii-printable.rs | 8 ++-- tests/base64.rs | 8 ++-- tests/chinese.rs | 8 ++-- tests/emoji.rs | 8 ++-- tests/ff.rs | 8 ++-- tests/random.rs | 8 ++-- tests/zero.rs | 8 ++-- 18 files changed, 180 insertions(+), 67 deletions(-) create mode 100644 compress/tests/zstd_interop.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ebe1ed..9b8c76d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,8 @@ jobs: - "" - "--no-default-features --features deflate" - "--no-default-features --features zstd" + - "--no-default-features --features zstd-rust" + - "--features zstd-rust" steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -52,7 +54,7 @@ jobs: default: true - name: assert deps don't exist with --no-default-features run: | - for DEP in libflate zstd; do + for DEP in libflate zstd structured-zstd; do if cargo tree --no-default-features --invert $DEP; then echo "Dependency $DEP should not exist with --no-default-features" exit 1 @@ -70,3 +72,11 @@ jobs: echo "Feature `zstd` should not create dep on `libflate`" exit 1 fi + - name: assert `zstd-rust` feature doesn't create dep on the C zstd bindings + run: | + for DEP in zstd zstd-sys libflate; do + if cargo tree --no-default-features --features zstd-rust --invert $DEP; then + echo "Feature `zstd-rust` should not create dep on $DEP" + exit 1 + fi + done diff --git a/Cargo.toml b/Cargo.toml index 81fe4fc..bd61fa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ license = "Apache-2.0" [workspace.dependencies] libflate = "2.3.0" zstd = "0.14.0" +structured-zstd = "0.0.56" [package] name = "include-flate" @@ -34,4 +35,7 @@ include-flate-compress = { version = "0.3.4", path = "compress", default-feature default = ["deflate", "zstd"] deflate = ["include-flate-codegen/deflate", "include-flate-compress/deflate"] zstd = ["include-flate-codegen/zstd", "include-flate-compress/zstd"] +# Zstandard through the pure-Rust `structured-zstd`, with no C toolchain. The +# `with zstd` syntax and the compressed format are the same as with `zstd`. +zstd-rust = ["include-flate-codegen/zstd-rust", "include-flate-compress/zstd-rust"] no-compression-warnings = ["include-flate-codegen/no-compression-warnings"] diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 0f0f4b3..889a2ce 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -22,4 +22,5 @@ proc-macro-error3 = "3.0.2" default = ["deflate", "zstd"] deflate = ["include-flate-compress/deflate"] zstd = ["include-flate-compress/zstd"] +zstd-rust = ["include-flate-compress/zstd-rust"] no-compression-warnings = [] diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index bfd379d..ddae492 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -98,13 +98,16 @@ impl syn::parse::Parse for FlateArgs { "Please enable the `deflate` feature", )); } else if lookahead.peek(kw::zstd) { - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] { input.parse::()?; Some(CompressionMethodTy(CompressionMethod::Zstd)) } - #[cfg(not(feature = "zstd"))] - return Err(Error::new(input.span(), "Please enable the `zstd` feature")); + #[cfg(not(any(feature = "zstd", feature = "zstd-rust")))] + return Err(Error::new( + input.span(), + "Please enable the `zstd` or `zstd-rust` feature", + )); } else { return Err(lookahead.error()); } diff --git a/compress/Cargo.toml b/compress/Cargo.toml index 2e6a4f5..89c7cfe 100644 --- a/compress/Cargo.toml +++ b/compress/Cargo.toml @@ -11,8 +11,12 @@ description = "Compression algorithm provider" [dependencies] libflate = { workspace = true, optional = true } zstd = { workspace = true, optional = true } +structured-zstd = { workspace = true, optional = true } [features] default = ["deflate", "zstd"] deflate = ["dep:libflate"] zstd = ["dep:zstd"] +# Zstandard through the pure-Rust `structured-zstd` instead of the C library. +# Same frame format; takes precedence when `zstd` is enabled as well. +zstd-rust = ["dep:structured-zstd"] diff --git a/compress/src/lib.rs b/compress/src/lib.rs index e6b0a47..acf0682 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -13,8 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(not(any(feature = "zstd", feature = "deflate")))] -compile_error!("You must enable either the `deflate` or `zstd` feature."); +#[cfg(not(any(feature = "zstd", feature = "zstd-rust", feature = "deflate")))] +compile_error!("You must enable either the `deflate`, `zstd` or `zstd-rust` feature."); use std::{ fmt, @@ -25,16 +25,49 @@ use std::{ use libflate::deflate::Decoder as DeflateDecoder; #[cfg(feature = "deflate")] use libflate::deflate::Encoder as DeflateEncoder; -#[cfg(feature = "zstd")] -use zstd::Decoder as ZstdDecoder; -#[cfg(feature = "zstd")] -use zstd::Encoder as ZstdEncoder; + +// Zstandard has two interchangeable backends producing and reading the same +// frame format: the `zstd` crate (bindings to the C library) and the pure-Rust +// `structured-zstd`. When both features are enabled, `zstd-rust` is used: a +// dependency that asks for it opts out of the C toolchain. +#[cfg(feature = "zstd-rust")] +type ZstdEncoder = structured_zstd::encoding::StreamingEncoder; +#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] +type ZstdEncoder = zstd::Encoder<'static, W>; +#[cfg(feature = "zstd-rust")] +type ZstdDecoder = + structured_zstd::decoding::StreamingDecoder; +#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] +type ZstdDecoder = zstd::Decoder<'static, std::io::BufReader>; + +#[cfg(feature = "zstd-rust")] +fn zstd_encoder(write: W) -> io::Result> { + // `Default` is this backend's counterpart of zstd level 3, which the C + // backend selects through level 0. + Ok(ZstdEncoder::new( + write, + structured_zstd::encoding::CompressionLevel::Default, + )) +} +#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] +fn zstd_encoder(write: W) -> io::Result> { + ZstdEncoder::new(write, 0) +} + +#[cfg(feature = "zstd-rust")] +fn zstd_decoder(read: R) -> io::Result> { + ZstdDecoder::new(read).map_err(io::Error::other) +} +#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] +fn zstd_decoder(read: R) -> io::Result> { + ZstdDecoder::new(read) +} #[derive(Debug)] pub enum CompressionError { #[cfg(feature = "deflate")] DeflateError(io::Error), - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] ZstdError(io::Error), IoError(io::Error), } @@ -50,7 +83,7 @@ impl fmt::Display for CompressionError { match self { #[cfg(feature = "deflate")] CompressionError::DeflateError(err) => write!(f, "Deflate error: {}", err), - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] CompressionError::ZstdError(err) => write!(f, "Zstd error: {}", err), CompressionError::IoError(err) => write!(f, "I/O error: {}", err), } @@ -61,7 +94,7 @@ impl fmt::Display for CompressionError { pub enum CompressionMethod { #[cfg(feature = "deflate")] Deflate, - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] Zstd, } @@ -79,14 +112,14 @@ impl CompressionMethod { clippy::derivable_impls, reason = "cfg_attr on defaults could be confusing" )] -#[cfg(any(feature = "deflate", feature = "zstd"))] +#[cfg(any(feature = "deflate", feature = "zstd", feature = "zstd-rust"))] impl Default for CompressionMethod { fn default() -> Self { #[cfg(feature = "deflate")] { Self::Deflate } - #[cfg(all(not(feature = "deflate"), feature = "zstd"))] + #[cfg(all(not(feature = "deflate"), any(feature = "zstd", feature = "zstd-rust")))] { Self::Zstd } @@ -98,7 +131,7 @@ impl fmt::Display for CompressionMethod { f.write_str(match self { #[cfg(feature = "deflate")] Self::Deflate => "deflate", - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] Self::Zstd => "zstd", }) } @@ -107,8 +140,8 @@ impl fmt::Display for CompressionMethod { pub enum FlateEncoder { #[cfg(feature = "deflate")] Deflate(DeflateEncoder), - #[cfg(feature = "zstd")] - Zstd(ZstdEncoder<'static, W>), + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + Zstd(ZstdEncoder), } impl FlateEncoder { @@ -116,8 +149,8 @@ impl FlateEncoder { match method { #[cfg(feature = "deflate")] CompressionMethod::Deflate => Ok(FlateEncoder::Deflate(DeflateEncoder::new(write))), - #[cfg(feature = "zstd")] - CompressionMethod::Zstd => ZstdEncoder::new(write, 0) + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + CompressionMethod::Zstd => zstd_encoder(write) .map(FlateEncoder::Zstd) .map_err(CompressionError::ZstdError), } @@ -129,7 +162,7 @@ impl Write for FlateEncoder { match self { #[cfg(feature = "deflate")] FlateEncoder::Deflate(encoder) => encoder.write(buf), - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] FlateEncoder::Zstd(encoder) => encoder.write(buf), } } @@ -138,7 +171,7 @@ impl Write for FlateEncoder { match self { #[cfg(feature = "deflate")] FlateEncoder::Deflate(encoder) => encoder.flush(), - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] FlateEncoder::Zstd(encoder) => encoder.flush(), } } @@ -152,17 +185,17 @@ impl FlateEncoder { .finish() .into_result() .map_err(CompressionError::DeflateError), - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] FlateEncoder::Zstd(encoder) => encoder.finish().map_err(CompressionError::ZstdError), } } } -pub enum FlateDecoder { +pub enum FlateDecoder { #[cfg(feature = "deflate")] Deflate(DeflateDecoder), - #[cfg(feature = "zstd")] - Zstd(ZstdDecoder<'static, std::io::BufReader>), + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + Zstd(ZstdDecoder), } impl FlateDecoder { @@ -170,9 +203,9 @@ impl FlateDecoder { match method { #[cfg(feature = "deflate")] CompressionMethod::Deflate => Ok(FlateDecoder::Deflate(DeflateDecoder::new(read))), - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] CompressionMethod::Zstd => { - let decoder = ZstdDecoder::new(read)?; + let decoder = zstd_decoder(read)?; Ok(FlateDecoder::Zstd(decoder)) } } @@ -184,7 +217,7 @@ impl Read for FlateDecoder { match self { #[cfg(feature = "deflate")] FlateDecoder::Deflate(decoder) => decoder.read(buf), - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] FlateDecoder::Zstd(decoder) => decoder.read(buf), } } diff --git a/compress/tests/zstd_interop.rs b/compress/tests/zstd_interop.rs new file mode 100644 index 0000000..74c80b7 --- /dev/null +++ b/compress/tests/zstd_interop.rs @@ -0,0 +1,53 @@ +// include-flate +// Copyright (C) SOFe +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The two Zstandard backends read each other's output: a crate built with +//! `zstd-rust` decodes what a `zstd` build embedded, and the reverse. + +#![cfg(all(feature = "zstd", feature = "zstd-rust"))] + +use std::io::Read; + +use include_flate_compress::{CompressionMethod, apply_compression, apply_decompression}; + +fn sample() -> Vec { + (0..200_000u32) + .flat_map(|i| format!("{} {} ", i % 977, i % 13).into_bytes()) + .collect() +} + +#[test] +fn a_c_zstd_frame_decodes_with_the_rust_backend() { + let data = sample(); + let frame = zstd::encode_all(&data[..], 0).unwrap(); + + let mut decoded = Vec::new(); + apply_decompression(&frame[..], &mut decoded, CompressionMethod::Zstd).unwrap(); + assert_eq!(decoded, data); +} + +#[test] +fn a_rust_backend_frame_decodes_with_c_zstd() { + let data = sample(); + let mut frame = Vec::new(); + apply_compression(&mut &data[..], &mut frame, CompressionMethod::Zstd).unwrap(); + + let mut decoded = Vec::new(); + zstd::Decoder::new(&frame[..]) + .unwrap() + .read_to_end(&mut decoded) + .unwrap(); + assert_eq!(decoded, data); +} diff --git a/src/lib.rs b/src/lib.rs index 764310b..aa6aa64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,7 +61,11 @@ pub use include_flate_compress::CompressionMethod; /// - If `$type` is `str` but the file is not fully valid UTF-8 /// /// # Algorithm -/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5]. +/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5] by default, +/// or Zstandard with `with zstd`. Zstandard is provided by the C library through the `zstd` +/// feature, or by the pure-Rust [`structured-zstd`][7] through the `zstd-rust` feature, which +/// needs no C toolchain and takes precedence when both are enabled. The two produce and read +/// the same format. /// /// # Examples /// Below are some basic examples. For actual compiled examples, see the [`tests`][6] directory. @@ -88,6 +92,7 @@ pub use include_flate_compress::CompressionMethod; /// [4]: https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates /// [5]: https://docs.rs/libflate/0.1.26/libflate/ /// [6]: https://github.com/SOF3/include-flate/tree/master/tests +/// [7]: https://docs.rs/structured-zstd #[macro_export] macro_rules! flate { ($(#[$meta:meta])* diff --git a/test_util.rs b/test_util.rs index eb42d06..9314a9f 100644 --- a/test_util.rs +++ b/test_util.rs @@ -60,7 +60,7 @@ pub fn verify_compression>(name: P, data: &[u8], method: Compress pub fn verify>(name: P, data: &[u8]) { #[cfg(feature = "deflate")] verify_compression(&name, data, CompressionMethod::Deflate); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_compression(&name, data, CompressionMethod::Zstd); assert_eq!(read_file(&name), data); } diff --git a/tests/009f.rs b/tests/009f.rs index 72cc022..134f0db 100644 --- a/tests/009f.rs +++ b/tests/009f.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/009f.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/009f.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: [u8] from "assets/009f.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/009f.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/009f.dat" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify("009f.dat", &DATA1); #[cfg(feature = "deflate")] verify("009f.dat", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify("009f.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("009f.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("009f.dat", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/ascii-control.rs b/tests/ascii-control.rs index bbf9b0e..772b595 100644 --- a/tests/ascii-control.rs +++ b/tests/ascii-control.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/ascii-control.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/ascii-control.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: str from "assets/ascii-control.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/ascii-control.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/ascii-control.txt" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify_str("ascii-control.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("ascii-control.txt", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_str("ascii-control.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("ascii-control.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("ascii-control.txt", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/ascii-printable.rs b/tests/ascii-printable.rs index db46312..f8a6500 100644 --- a/tests/ascii-printable.rs +++ b/tests/ascii-printable.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/ascii-printable.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/ascii-printable.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: str from "assets/ascii-printable.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/ascii-printable.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/ascii-printable.txt" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify_str("ascii-printable.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("ascii-printable.txt", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_str("ascii-printable.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("ascii-printable.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("ascii-printable.txt", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/base64.rs b/tests/base64.rs index 5171c3f..41e62b5 100644 --- a/tests/base64.rs +++ b/tests/base64.rs @@ -23,11 +23,11 @@ pub static DATA_RAW: &[u8] = flate!(pub static DATA1: str from "assets/base64.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/base64.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: str from "assets/base64.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/base64.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/base64.txt" with zstd); #[test] @@ -35,10 +35,10 @@ fn test() { verify_str("base64.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("base64.txt", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_str("base64.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("base64.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("base64.txt", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/chinese.rs b/tests/chinese.rs index a092f05..5afa30e 100644 --- a/tests/chinese.rs +++ b/tests/chinese.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/chinese.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/chinese.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: str from "assets/chinese.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/chinese.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/chinese.txt" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify_str("chinese.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("chinese.txt", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_str("chinese.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("chinese.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("chinese.txt", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/emoji.rs b/tests/emoji.rs index 24efc71..964879c 100644 --- a/tests/emoji.rs +++ b/tests/emoji.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/emoji.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/emoji.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: str from "assets/emoji.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/emoji.txt" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/emoji.txt" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify_str("emoji.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("emoji.txt", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_str("emoji.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("emoji.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("emoji.txt", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/ff.rs b/tests/ff.rs index 8bc13e1..e09635f 100644 --- a/tests/ff.rs +++ b/tests/ff.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/ff.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/ff.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: [u8] from "assets/ff.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/ff.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/ff.dat" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify("ff.dat", &DATA1); #[cfg(feature = "deflate")] verify("ff.dat", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify("ff.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("ff.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("ff.dat", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/random.rs b/tests/random.rs index e1254e0..e89f08d 100644 --- a/tests/random.rs +++ b/tests/random.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/random.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/random.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: [u8] from "assets/random.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/random.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/random.dat" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify("random.dat", &DATA1); #[cfg(feature = "deflate")] verify("random.dat", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify("random.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("random.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("random.dat", CompressionMethod::Zstd, &DATA5); } diff --git a/tests/zero.rs b/tests/zero.rs index add2ea9..8a3eff0 100644 --- a/tests/zero.rs +++ b/tests/zero.rs @@ -20,11 +20,11 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/zero.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/zero.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA3: [u8] from "assets/zero.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/zero.dat" with deflate); -#[cfg(feature = "zstd")] +#[cfg(any(feature = "zstd", feature = "zstd-rust"))] flate!(pub static DATA5: IFlate from "assets/zero.dat" with zstd); #[test] @@ -32,10 +32,10 @@ fn test() { verify("zero.dat", &DATA1); #[cfg(feature = "deflate")] verify("zero.dat", &DATA2); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify("zero.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("zero.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(feature = "zstd")] + #[cfg(any(feature = "zstd", feature = "zstd-rust"))] verify_iflate("zero.dat", CompressionMethod::Zstd, &DATA5); } From 43461ade488f1d211eaa974c10339e9391030d3a Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 26 Sep 2026 13:03:34 +0300 Subject: [PATCH 2/9] Box the pure-Rust Zstandard coders structured-zstd keeps the coder state inline, which made the Zstd variants of FlateEncoder and FlateDecoder far larger than the others (clippy::large_enum_variant). Boxing them keeps the enums compact; the C backend already holds its state behind a pointer. --- compress/src/lib.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/compress/src/lib.rs b/compress/src/lib.rs index acf0682..ca46241 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -30,13 +30,17 @@ use libflate::deflate::Encoder as DeflateEncoder; // frame format: the `zstd` crate (bindings to the C library) and the pure-Rust // `structured-zstd`. When both features are enabled, `zstd-rust` is used: a // dependency that asks for it opts out of the C toolchain. +// +// The pure-Rust coder keeps its state inline (the C one holds a pointer to +// it), so it is boxed to keep the enums below the size of their other +// variants. #[cfg(feature = "zstd-rust")] -type ZstdEncoder = structured_zstd::encoding::StreamingEncoder; +type ZstdEncoder = Box>; #[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] type ZstdEncoder = zstd::Encoder<'static, W>; #[cfg(feature = "zstd-rust")] type ZstdDecoder = - structured_zstd::decoding::StreamingDecoder; + Box>; #[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] type ZstdDecoder = zstd::Decoder<'static, std::io::BufReader>; @@ -44,10 +48,10 @@ type ZstdDecoder = zstd::Decoder<'static, std::io::BufReader>; fn zstd_encoder(write: W) -> io::Result> { // `Default` is this backend's counterpart of zstd level 3, which the C // backend selects through level 0. - Ok(ZstdEncoder::new( + Ok(Box::new(structured_zstd::encoding::StreamingEncoder::new( write, structured_zstd::encoding::CompressionLevel::Default, - )) + ))) } #[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] fn zstd_encoder(write: W) -> io::Result> { @@ -56,7 +60,9 @@ fn zstd_encoder(write: W) -> io::Result> { #[cfg(feature = "zstd-rust")] fn zstd_decoder(read: R) -> io::Result> { - ZstdDecoder::new(read).map_err(io::Error::other) + structured_zstd::decoding::StreamingDecoder::new(read) + .map(Box::new) + .map_err(io::Error::other) } #[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] fn zstd_decoder(read: R) -> io::Result> { @@ -185,7 +191,9 @@ impl FlateEncoder { .finish() .into_result() .map_err(CompressionError::DeflateError), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd-rust")] + FlateEncoder::Zstd(encoder) => (*encoder).finish().map_err(CompressionError::ZstdError), + #[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] FlateEncoder::Zstd(encoder) => encoder.finish().map_err(CompressionError::ZstdError), } } From 9a485dfe68342013822849e08f303bfe1144ed46 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 6 Oct 2026 11:03:04 +0300 Subject: [PATCH 3/9] Expose the pure-Rust Zstandard backend as its own method zstd-rust used to replace the C backend behind CompressionMethod::Zstd whenever both features were enabled. Feature unification made that a global switch: any crate in the graph enabling zstd-rust changed the backend of every other crate's `with zstd`, which is not additive. zstd-rust now adds CompressionMethod::ZstdRust, selected by `with zstd_rust`, next to Deflate and Zstd. Each method is gated by its own feature only, and both Zstandard methods can be enabled together. Frames stay interchangeable between them. - compress: ZstdRust encoder/decoder variants and ZstdRustError; Zstd is back to the upstream code. Default prefers deflate, then zstd, then zstd-rust. - codegen, root crate: zstd_rust keyword and __parse_algo arm. - Tests: every asset test also embeds with zstd_rust. verify() encodes with each Zstandard method and decodes with the other when both are enabled; the interop check is now a unit test of the compress crate. - test_util: unwrap_or instead of a lazy closure (clippy). - structured-zstd 0.0.56 -> 0.0.58. --- Cargo.toml | 6 +- codegen/src/lib.rs | 16 ++- compress/Cargo.toml | 4 +- compress/src/lib.rs | 127 ++++++++++-------- .../{tests/zstd_interop.rs => src/tests.rs} | 32 ++--- src/lib.rs | 13 +- test_util.rs | 22 ++- tests/009f.rs | 16 ++- tests/ascii-control.rs | 16 ++- tests/ascii-printable.rs | 16 ++- tests/base64.rs | 16 ++- tests/chinese.rs | 16 ++- tests/emoji.rs | 16 ++- tests/ff.rs | 16 ++- tests/random.rs | 16 ++- tests/zero.rs | 16 ++- 16 files changed, 237 insertions(+), 127 deletions(-) rename compress/{tests/zstd_interop.rs => src/tests.rs} (54%) diff --git a/Cargo.toml b/Cargo.toml index bd61fa4..bee1b12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ license = "Apache-2.0" [workspace.dependencies] libflate = "2.3.0" zstd = "0.14.0" -structured-zstd = "0.0.56" +structured-zstd = "0.0.58" [package] name = "include-flate" @@ -35,7 +35,7 @@ include-flate-compress = { version = "0.3.4", path = "compress", default-feature default = ["deflate", "zstd"] deflate = ["include-flate-codegen/deflate", "include-flate-compress/deflate"] zstd = ["include-flate-codegen/zstd", "include-flate-compress/zstd"] -# Zstandard through the pure-Rust `structured-zstd`, with no C toolchain. The -# `with zstd` syntax and the compressed format are the same as with `zstd`. +# Zstandard through the pure-Rust `structured-zstd`, with no C toolchain, +# selected by `with zstd_rust`. Frames are interchangeable with `with zstd`. zstd-rust = ["include-flate-codegen/zstd-rust", "include-flate-compress/zstd-rust"] no-compression-warnings = ["include-flate-codegen/no-compression-warnings"] diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index ddae492..710ff69 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -72,6 +72,7 @@ pub fn deflate_utf8_file(ts: TokenStream) -> TokenStream { /// flate!(pub static DATA: [u8] from "assets/009f.dat"); // default, DEFLATE /// flate!(pub static DATA: [u8] from "assets/009f.dat" with zstd); // Use Zstd for this file spcifically /// flate!(pub static DATA: [u8] from "assets/009f.dat" with deflate); // Explicitly use DEFLATE. +/// flate!(pub static DATA: [u8] from "assets/009f.dat" with zstd_rust); // Zstd without the C library /// ``` struct FlateArgs { path: syn::LitStr, @@ -98,15 +99,23 @@ impl syn::parse::Parse for FlateArgs { "Please enable the `deflate` feature", )); } else if lookahead.peek(kw::zstd) { - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] { input.parse::()?; Some(CompressionMethodTy(CompressionMethod::Zstd)) } - #[cfg(not(any(feature = "zstd", feature = "zstd-rust")))] + #[cfg(not(feature = "zstd"))] + return Err(Error::new(input.span(), "Please enable the `zstd` feature")); + } else if lookahead.peek(kw::zstd_rust) { + #[cfg(feature = "zstd-rust")] + { + input.parse::()?; + Some(CompressionMethodTy(CompressionMethod::ZstdRust)) + } + #[cfg(not(feature = "zstd-rust"))] return Err(Error::new( input.span(), - "Please enable the `zstd` or `zstd-rust` feature", + "Please enable the `zstd-rust` feature", )); } else { return Err(lookahead.error()); @@ -120,6 +129,7 @@ impl syn::parse::Parse for FlateArgs { mod kw { syn::custom_keyword!(deflate); syn::custom_keyword!(zstd); + syn::custom_keyword!(zstd_rust); } #[derive(Debug, Default)] diff --git a/compress/Cargo.toml b/compress/Cargo.toml index 89c7cfe..cdd2277 100644 --- a/compress/Cargo.toml +++ b/compress/Cargo.toml @@ -17,6 +17,6 @@ structured-zstd = { workspace = true, optional = true } default = ["deflate", "zstd"] deflate = ["dep:libflate"] zstd = ["dep:zstd"] -# Zstandard through the pure-Rust `structured-zstd` instead of the C library. -# Same frame format; takes precedence when `zstd` is enabled as well. +# Zstandard through the pure-Rust `structured-zstd`, as `CompressionMethod::ZstdRust`. +# Same frame format as `CompressionMethod::Zstd`. zstd-rust = ["dep:structured-zstd"] diff --git a/compress/src/lib.rs b/compress/src/lib.rs index ca46241..731ffb6 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -25,56 +25,28 @@ use std::{ use libflate::deflate::Decoder as DeflateDecoder; #[cfg(feature = "deflate")] use libflate::deflate::Encoder as DeflateEncoder; +#[cfg(feature = "zstd")] +use zstd::Decoder as ZstdDecoder; +#[cfg(feature = "zstd")] +use zstd::Encoder as ZstdEncoder; -// Zstandard has two interchangeable backends producing and reading the same -// frame format: the `zstd` crate (bindings to the C library) and the pure-Rust -// `structured-zstd`. When both features are enabled, `zstd-rust` is used: a -// dependency that asks for it opts out of the C toolchain. -// -// The pure-Rust coder keeps its state inline (the C one holds a pointer to -// it), so it is boxed to keep the enums below the size of their other +// The pure-Rust coders keep their state inline (the C ones hold a pointer to +// it), so they are boxed to keep the enums below the size of their other // variants. #[cfg(feature = "zstd-rust")] -type ZstdEncoder = Box>; -#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] -type ZstdEncoder = zstd::Encoder<'static, W>; +type ZstdRustEncoder = Box>; #[cfg(feature = "zstd-rust")] -type ZstdDecoder = +type ZstdRustDecoder = Box>; -#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] -type ZstdDecoder = zstd::Decoder<'static, std::io::BufReader>; - -#[cfg(feature = "zstd-rust")] -fn zstd_encoder(write: W) -> io::Result> { - // `Default` is this backend's counterpart of zstd level 3, which the C - // backend selects through level 0. - Ok(Box::new(structured_zstd::encoding::StreamingEncoder::new( - write, - structured_zstd::encoding::CompressionLevel::Default, - ))) -} -#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] -fn zstd_encoder(write: W) -> io::Result> { - ZstdEncoder::new(write, 0) -} - -#[cfg(feature = "zstd-rust")] -fn zstd_decoder(read: R) -> io::Result> { - structured_zstd::decoding::StreamingDecoder::new(read) - .map(Box::new) - .map_err(io::Error::other) -} -#[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] -fn zstd_decoder(read: R) -> io::Result> { - ZstdDecoder::new(read) -} #[derive(Debug)] pub enum CompressionError { #[cfg(feature = "deflate")] DeflateError(io::Error), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] ZstdError(io::Error), + #[cfg(feature = "zstd-rust")] + ZstdRustError(io::Error), IoError(io::Error), } @@ -89,19 +61,26 @@ impl fmt::Display for CompressionError { match self { #[cfg(feature = "deflate")] CompressionError::DeflateError(err) => write!(f, "Deflate error: {}", err), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] CompressionError::ZstdError(err) => write!(f, "Zstd error: {}", err), + #[cfg(feature = "zstd-rust")] + CompressionError::ZstdRustError(err) => write!(f, "Zstd (pure Rust) error: {}", err), CompressionError::IoError(err) => write!(f, "I/O error: {}", err), } } } +/// `Zstd` and `ZstdRust` produce and read the same Zstandard frame format: +/// `Zstd` through the C library, `ZstdRust` through the pure-Rust +/// `structured-zstd`, which needs no C toolchain. #[derive(Debug, Copy, Clone, PartialEq)] pub enum CompressionMethod { #[cfg(feature = "deflate")] Deflate, - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] Zstd, + #[cfg(feature = "zstd-rust")] + ZstdRust, } impl CompressionMethod { @@ -125,10 +104,14 @@ impl Default for CompressionMethod { { Self::Deflate } - #[cfg(all(not(feature = "deflate"), any(feature = "zstd", feature = "zstd-rust")))] + #[cfg(all(not(feature = "deflate"), feature = "zstd"))] { Self::Zstd } + #[cfg(all(not(feature = "deflate"), not(feature = "zstd"), feature = "zstd-rust"))] + { + Self::ZstdRust + } } } @@ -137,8 +120,10 @@ impl fmt::Display for CompressionMethod { f.write_str(match self { #[cfg(feature = "deflate")] Self::Deflate => "deflate", - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] Self::Zstd => "zstd", + #[cfg(feature = "zstd-rust")] + Self::ZstdRust => "zstd_rust", }) } } @@ -146,8 +131,10 @@ impl fmt::Display for CompressionMethod { pub enum FlateEncoder { #[cfg(feature = "deflate")] Deflate(DeflateEncoder), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] - Zstd(ZstdEncoder), + #[cfg(feature = "zstd")] + Zstd(ZstdEncoder<'static, W>), + #[cfg(feature = "zstd-rust")] + ZstdRust(ZstdRustEncoder), } impl FlateEncoder { @@ -155,10 +142,19 @@ impl FlateEncoder { match method { #[cfg(feature = "deflate")] CompressionMethod::Deflate => Ok(FlateEncoder::Deflate(DeflateEncoder::new(write))), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] - CompressionMethod::Zstd => zstd_encoder(write) + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => ZstdEncoder::new(write, 0) .map(FlateEncoder::Zstd) .map_err(CompressionError::ZstdError), + // `Default` is this backend's counterpart of zstd level 3, which + // the C backend selects through level 0. + #[cfg(feature = "zstd-rust")] + CompressionMethod::ZstdRust => Ok(FlateEncoder::ZstdRust(Box::new( + structured_zstd::encoding::StreamingEncoder::new( + write, + structured_zstd::encoding::CompressionLevel::Default, + ), + ))), } } } @@ -168,8 +164,10 @@ impl Write for FlateEncoder { match self { #[cfg(feature = "deflate")] FlateEncoder::Deflate(encoder) => encoder.write(buf), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.write(buf), + #[cfg(feature = "zstd-rust")] + FlateEncoder::ZstdRust(encoder) => encoder.write(buf), } } @@ -177,8 +175,10 @@ impl Write for FlateEncoder { match self { #[cfg(feature = "deflate")] FlateEncoder::Deflate(encoder) => encoder.flush(), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.flush(), + #[cfg(feature = "zstd-rust")] + FlateEncoder::ZstdRust(encoder) => encoder.flush(), } } } @@ -191,10 +191,12 @@ impl FlateEncoder { .finish() .into_result() .map_err(CompressionError::DeflateError), - #[cfg(feature = "zstd-rust")] - FlateEncoder::Zstd(encoder) => (*encoder).finish().map_err(CompressionError::ZstdError), - #[cfg(all(feature = "zstd", not(feature = "zstd-rust")))] + #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.finish().map_err(CompressionError::ZstdError), + #[cfg(feature = "zstd-rust")] + FlateEncoder::ZstdRust(encoder) => { + (*encoder).finish().map_err(CompressionError::ZstdRustError) + } } } } @@ -202,8 +204,10 @@ impl FlateEncoder { pub enum FlateDecoder { #[cfg(feature = "deflate")] Deflate(DeflateDecoder), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] - Zstd(ZstdDecoder), + #[cfg(feature = "zstd")] + Zstd(ZstdDecoder<'static, std::io::BufReader>), + #[cfg(feature = "zstd-rust")] + ZstdRust(ZstdRustDecoder), } impl FlateDecoder { @@ -211,11 +215,15 @@ impl FlateDecoder { match method { #[cfg(feature = "deflate")] CompressionMethod::Deflate => Ok(FlateDecoder::Deflate(DeflateDecoder::new(read))), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] CompressionMethod::Zstd => { - let decoder = zstd_decoder(read)?; + let decoder = ZstdDecoder::new(read)?; Ok(FlateDecoder::Zstd(decoder)) } + #[cfg(feature = "zstd-rust")] + CompressionMethod::ZstdRust => structured_zstd::decoding::StreamingDecoder::new(read) + .map(|decoder| FlateDecoder::ZstdRust(Box::new(decoder))) + .map_err(|err| CompressionError::ZstdRustError(io::Error::other(err))), } } } @@ -225,8 +233,10 @@ impl Read for FlateDecoder { match self { #[cfg(feature = "deflate")] FlateDecoder::Deflate(decoder) => decoder.read(buf), - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] FlateDecoder::Zstd(decoder) => decoder.read(buf), + #[cfg(feature = "zstd-rust")] + FlateDecoder::ZstdRust(decoder) => decoder.read(buf), } } } @@ -254,3 +264,6 @@ pub fn apply_decompression( io::copy(&mut decoder, &mut writer)?; Ok(()) } + +#[cfg(test)] +mod tests; diff --git a/compress/tests/zstd_interop.rs b/compress/src/tests.rs similarity index 54% rename from compress/tests/zstd_interop.rs rename to compress/src/tests.rs index 74c80b7..1e94290 100644 --- a/compress/tests/zstd_interop.rs +++ b/compress/src/tests.rs @@ -13,14 +13,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! The two Zstandard backends read each other's output: a crate built with -//! `zstd-rust` decodes what a `zstd` build embedded, and the reverse. +//! `Zstd` and `ZstdRust` share the Zstandard frame format, so each one +//! decodes what the other encoded. #![cfg(all(feature = "zstd", feature = "zstd-rust"))] -use std::io::Read; - -use include_flate_compress::{CompressionMethod, apply_compression, apply_decompression}; +use super::{CompressionMethod, apply_compression, apply_decompression}; fn sample() -> Vec { (0..200_000u32) @@ -28,26 +26,22 @@ fn sample() -> Vec { .collect() } -#[test] -fn a_c_zstd_frame_decodes_with_the_rust_backend() { +fn round_trip(encode: CompressionMethod, decode: CompressionMethod) { let data = sample(); - let frame = zstd::encode_all(&data[..], 0).unwrap(); + let mut frame = Vec::new(); + apply_compression(&mut &data[..], &mut frame, encode).unwrap(); let mut decoded = Vec::new(); - apply_decompression(&frame[..], &mut decoded, CompressionMethod::Zstd).unwrap(); + apply_decompression(&frame[..], &mut decoded, decode).unwrap(); assert_eq!(decoded, data); } #[test] -fn a_rust_backend_frame_decodes_with_c_zstd() { - let data = sample(); - let mut frame = Vec::new(); - apply_compression(&mut &data[..], &mut frame, CompressionMethod::Zstd).unwrap(); +fn a_zstd_frame_decodes_with_zstd_rust() { + round_trip(CompressionMethod::Zstd, CompressionMethod::ZstdRust); +} - let mut decoded = Vec::new(); - zstd::Decoder::new(&frame[..]) - .unwrap() - .read_to_end(&mut decoded) - .unwrap(); - assert_eq!(decoded, data); +#[test] +fn a_zstd_rust_frame_decodes_with_zstd() { + round_trip(CompressionMethod::ZstdRust, CompressionMethod::Zstd); } diff --git a/src/lib.rs b/src/lib.rs index aa6aa64..cd192b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,11 +61,11 @@ pub use include_flate_compress::CompressionMethod; /// - If `$type` is `str` but the file is not fully valid UTF-8 /// /// # Algorithm -/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5] by default, -/// or Zstandard with `with zstd`. Zstandard is provided by the C library through the `zstd` -/// feature, or by the pure-Rust [`structured-zstd`][7] through the `zstd-rust` feature, which -/// needs no C toolchain and takes precedence when both are enabled. The two produce and read -/// the same format. +/// Compression and decompression use the DEFLATE algorithm from [`libflate`][5] by default. +/// `with zstd` selects Zstandard through the C library (the `zstd` feature), and +/// `with zstd_rust` selects Zstandard through the pure-Rust [`structured-zstd`][7] +/// (the `zstd-rust` feature), which needs no C toolchain. Both produce and read the same +/// frame format. /// /// # Examples /// Below are some basic examples. For actual compiled examples, see the [`tests`][6] directory. @@ -144,6 +144,9 @@ macro_rules! __parse_algo { (zstd) => { $crate::CompressionMethod::Zstd }; + (zstd_rust) => { + $crate::CompressionMethod::ZstdRust + }; ($other:ident) => { compile_error!("Unknown compression algorithm: {}", stringify!($other)) }; diff --git a/test_util.rs b/test_util.rs index 9314a9f..37cc336 100644 --- a/test_util.rs +++ b/test_util.rs @@ -23,7 +23,7 @@ use include_flate_compress::{CompressionMethod, apply_compression, apply_decompr pub fn get_file_path>(relative_from: Option<&Path>, path: P) -> PathBuf { let cargo_manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); let default_base_path = Path::new(&cargo_manifest_dir); - let base_path = relative_from.unwrap_or_else(|| default_base_path); + let base_path = relative_from.unwrap_or(default_base_path); base_path.join("assets").join(path) } @@ -57,11 +57,29 @@ pub fn verify_compression>(name: P, data: &[u8], method: Compress assert_ne!(compressed_buffer.as_slice(), decompressed_buffer.as_slice()); } +/// Encodes the file with `encode` and checks that `decode` restores it exactly. +#[cfg(all(feature = "zstd", feature = "zstd-rust"))] +pub fn verify_cross>(name: P, encode: CompressionMethod, decode: CompressionMethod) { + let original = read_file(&name); + let mut compressed = Vec::new(); + apply_compression(&mut &original[..], &mut compressed, encode).unwrap(); + let mut decompressed = Vec::new(); + apply_decompression(&compressed[..], &mut decompressed, decode).unwrap(); + assert_eq!(decompressed, original); +} + pub fn verify>(name: P, data: &[u8]) { #[cfg(feature = "deflate")] verify_compression(&name, data, CompressionMethod::Deflate); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_compression(&name, data, CompressionMethod::Zstd); + #[cfg(feature = "zstd-rust")] + verify_compression(&name, data, CompressionMethod::ZstdRust); + #[cfg(all(feature = "zstd", feature = "zstd-rust"))] + { + verify_cross(&name, CompressionMethod::Zstd, CompressionMethod::ZstdRust); + verify_cross(&name, CompressionMethod::ZstdRust, CompressionMethod::Zstd); + } assert_eq!(read_file(&name), data); } diff --git a/tests/009f.rs b/tests/009f.rs index 134f0db..188cb3d 100644 --- a/tests/009f.rs +++ b/tests/009f.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/009f.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/009f.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: [u8] from "assets/009f.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/009f.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/009f.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/009f.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/009f.dat" with zstd_rust); #[test] fn test() { verify("009f.dat", &DATA1); #[cfg(feature = "deflate")] verify("009f.dat", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify("009f.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("009f.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("009f.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("009f.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("009f.dat", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/ascii-control.rs b/tests/ascii-control.rs index 772b595..a1bd075 100644 --- a/tests/ascii-control.rs +++ b/tests/ascii-control.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/ascii-control.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/ascii-control.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: str from "assets/ascii-control.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/ascii-control.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/ascii-control.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/ascii-control.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/ascii-control.txt" with zstd_rust); #[test] fn test() { verify_str("ascii-control.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("ascii-control.txt", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_str("ascii-control.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("ascii-control.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("ascii-control.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("ascii-control.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("ascii-control.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/ascii-printable.rs b/tests/ascii-printable.rs index f8a6500..f315b8c 100644 --- a/tests/ascii-printable.rs +++ b/tests/ascii-printable.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/ascii-printable.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/ascii-printable.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: str from "assets/ascii-printable.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/ascii-printable.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/ascii-printable.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/ascii-printable.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/ascii-printable.txt" with zstd_rust); #[test] fn test() { verify_str("ascii-printable.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("ascii-printable.txt", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_str("ascii-printable.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("ascii-printable.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("ascii-printable.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("ascii-printable.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("ascii-printable.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/base64.rs b/tests/base64.rs index 41e62b5..acf3fc8 100644 --- a/tests/base64.rs +++ b/tests/base64.rs @@ -23,22 +23,30 @@ pub static DATA_RAW: &[u8] = flate!(pub static DATA1: str from "assets/base64.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/base64.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: str from "assets/base64.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/base64.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/base64.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/base64.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/base64.txt" with zstd_rust); #[test] fn test() { verify_str("base64.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("base64.txt", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_str("base64.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("base64.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("base64.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("base64.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("base64.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/chinese.rs b/tests/chinese.rs index 5afa30e..6e292a3 100644 --- a/tests/chinese.rs +++ b/tests/chinese.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/chinese.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/chinese.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: str from "assets/chinese.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/chinese.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/chinese.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/chinese.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/chinese.txt" with zstd_rust); #[test] fn test() { verify_str("chinese.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("chinese.txt", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_str("chinese.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("chinese.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("chinese.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("chinese.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("chinese.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/emoji.rs b/tests/emoji.rs index 964879c..835c625 100644 --- a/tests/emoji.rs +++ b/tests/emoji.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: str from "assets/emoji.txt"); #[cfg(feature = "deflate")] flate!(pub static DATA2: str from "assets/emoji.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: str from "assets/emoji.txt" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/emoji.txt" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/emoji.txt" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: str from "assets/emoji.txt" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/emoji.txt" with zstd_rust); #[test] fn test() { verify_str("emoji.txt", &DATA1); #[cfg(feature = "deflate")] verify_str("emoji.txt", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_str("emoji.txt", &DATA3); #[cfg(feature = "deflate")] verify_iflate("emoji.txt", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("emoji.txt", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify_str("emoji.txt", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("emoji.txt", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/ff.rs b/tests/ff.rs index e09635f..787e830 100644 --- a/tests/ff.rs +++ b/tests/ff.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/ff.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/ff.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: [u8] from "assets/ff.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/ff.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/ff.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/ff.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/ff.dat" with zstd_rust); #[test] fn test() { verify("ff.dat", &DATA1); #[cfg(feature = "deflate")] verify("ff.dat", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify("ff.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("ff.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("ff.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("ff.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("ff.dat", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/random.rs b/tests/random.rs index e89f08d..05ac978 100644 --- a/tests/random.rs +++ b/tests/random.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/random.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/random.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: [u8] from "assets/random.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/random.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/random.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/random.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/random.dat" with zstd_rust); #[test] fn test() { verify("random.dat", &DATA1); #[cfg(feature = "deflate")] verify("random.dat", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify("random.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("random.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("random.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("random.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("random.dat", CompressionMethod::ZstdRust, &DATA7); } diff --git a/tests/zero.rs b/tests/zero.rs index 8a3eff0..68239c0 100644 --- a/tests/zero.rs +++ b/tests/zero.rs @@ -20,22 +20,30 @@ use include_flate::flate; flate!(pub static DATA1: [u8] from "assets/zero.dat"); #[cfg(feature = "deflate")] flate!(pub static DATA2: [u8] from "assets/zero.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA3: [u8] from "assets/zero.dat" with zstd); #[cfg(feature = "deflate")] flate!(pub static DATA4: IFlate from "assets/zero.dat" with deflate); -#[cfg(any(feature = "zstd", feature = "zstd-rust"))] +#[cfg(feature = "zstd")] flate!(pub static DATA5: IFlate from "assets/zero.dat" with zstd); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA6: [u8] from "assets/zero.dat" with zstd_rust); +#[cfg(feature = "zstd-rust")] +flate!(pub static DATA7: IFlate from "assets/zero.dat" with zstd_rust); #[test] fn test() { verify("zero.dat", &DATA1); #[cfg(feature = "deflate")] verify("zero.dat", &DATA2); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify("zero.dat", &DATA3); #[cfg(feature = "deflate")] verify_iflate("zero.dat", CompressionMethod::Deflate, &DATA4); - #[cfg(any(feature = "zstd", feature = "zstd-rust"))] + #[cfg(feature = "zstd")] verify_iflate("zero.dat", CompressionMethod::Zstd, &DATA5); + #[cfg(feature = "zstd-rust")] + verify("zero.dat", &DATA6); + #[cfg(feature = "zstd-rust")] + verify_iflate("zero.dat", CompressionMethod::ZstdRust, &DATA7); } From 00424c53c1a184367e61b0d32bf3c05fd33e9474 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Tue, 6 Oct 2026 11:03:52 +0300 Subject: [PATCH 4/9] Keep structured-zstd at 0.0.56 0.0.57 and 0.0.58 change only the encoder, dictionary training and the CLI, and 0.0.58 decodes about 4% slower. include-flate decodes at run time, so the older release is the better pin. jieba-rs dictionary (5,071,843 B), one C-encoded frame, 200 interleaved decodes per decoder, two runs: 0.0.56: min 7.05 / 7.19 ms, median 9.05 / 7.86 ms 0.0.58: min 7.33 / 7.39 ms, median 9.90 / 8.18 ms --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index bee1b12..5ed1f32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ license = "Apache-2.0" [workspace.dependencies] libflate = "2.3.0" zstd = "0.14.0" -structured-zstd = "0.0.58" +structured-zstd = "0.0.56" [package] name = "include-flate" From a10a2f4afca03d3bda82bc57d946244ab89a1895 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 7 Oct 2026 18:35:15 +0300 Subject: [PATCH 5/9] Decode zstd_rust data in one call into an exact buffer The streaming decoder routes every sequence through its internal ring buffer and hands the output out through Read, which made zstd_rust decoding about 1.5x the C time on x86. The embedded data is always a whole frame in memory, so it can be decoded straight into the final buffer instead. - The zstd_rust encoder collects its input and writes it as one frame, so the frame declares Frame_Content_Size. - decompress_slice decodes a frame that declares its size with FrameDecoder::decode_all into an exactly sized Vec, and rejects a frame that holds less than it declares. A frame without the size still goes through the streaming decoder. - decode uses decompress_slice, so flate! and IFlate take this path. --- compress/src/lib.rs | 89 +++++++++++++++++++++++++------- compress/src/tests.rs | 115 ++++++++++++++++++++++++++++++++++++++---- src/lib.rs | 13 ++--- test_util.rs | 3 ++ 4 files changed, 184 insertions(+), 36 deletions(-) diff --git a/compress/src/lib.rs b/compress/src/lib.rs index 731ffb6..1fad765 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -30,15 +30,21 @@ use zstd::Decoder as ZstdDecoder; #[cfg(feature = "zstd")] use zstd::Encoder as ZstdEncoder; -// The pure-Rust coders keep their state inline (the C ones hold a pointer to -// it), so they are boxed to keep the enums below the size of their other -// variants. -#[cfg(feature = "zstd-rust")] -type ZstdRustEncoder = Box>; +// The pure-Rust decoder keeps its state inline (the C one holds a pointer to +// it), so it is boxed to keep the enum below the size of its other variants. #[cfg(feature = "zstd-rust")] type ZstdRustDecoder = Box>; +/// Collects the whole input and encodes it as one frame on finish, so the +/// frame declares `Frame_Content_Size` and `decompress_slice` can decode it +/// into an exactly sized buffer in one call. +#[cfg(feature = "zstd-rust")] +pub struct ZstdRustEncoder { + input: Vec, + sink: W, +} + #[derive(Debug)] pub enum CompressionError { #[cfg(feature = "deflate")] @@ -146,15 +152,11 @@ impl FlateEncoder { CompressionMethod::Zstd => ZstdEncoder::new(write, 0) .map(FlateEncoder::Zstd) .map_err(CompressionError::ZstdError), - // `Default` is this backend's counterpart of zstd level 3, which - // the C backend selects through level 0. #[cfg(feature = "zstd-rust")] - CompressionMethod::ZstdRust => Ok(FlateEncoder::ZstdRust(Box::new( - structured_zstd::encoding::StreamingEncoder::new( - write, - structured_zstd::encoding::CompressionLevel::Default, - ), - ))), + CompressionMethod::ZstdRust => Ok(FlateEncoder::ZstdRust(ZstdRustEncoder { + input: Vec::new(), + sink: write, + })), } } } @@ -167,7 +169,10 @@ impl Write for FlateEncoder { #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.write(buf), #[cfg(feature = "zstd-rust")] - FlateEncoder::ZstdRust(encoder) => encoder.write(buf), + FlateEncoder::ZstdRust(encoder) => { + encoder.input.extend_from_slice(buf); + Ok(buf.len()) + } } } @@ -177,8 +182,9 @@ impl Write for FlateEncoder { FlateEncoder::Deflate(encoder) => encoder.flush(), #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.flush(), + // Nothing is encoded before the frame is finished. #[cfg(feature = "zstd-rust")] - FlateEncoder::ZstdRust(encoder) => encoder.flush(), + FlateEncoder::ZstdRust(_) => Ok(()), } } } @@ -193,9 +199,17 @@ impl FlateEncoder { .map_err(CompressionError::DeflateError), #[cfg(feature = "zstd")] FlateEncoder::Zstd(encoder) => encoder.finish().map_err(CompressionError::ZstdError), + // `Default` is this backend's counterpart of zstd level 3, which + // the C backend selects through level 0. #[cfg(feature = "zstd-rust")] - FlateEncoder::ZstdRust(encoder) => { - (*encoder).finish().map_err(CompressionError::ZstdRustError) + FlateEncoder::ZstdRust(ZstdRustEncoder { input, mut sink }) => { + let frame = structured_zstd::encoding::compress_slice_to_vec( + &input, + structured_zstd::encoding::CompressionLevel::Default, + ); + sink.write_all(&frame) + .map_err(CompressionError::ZstdRustError)?; + Ok(sink) } } } @@ -265,5 +279,46 @@ pub fn apply_decompression( Ok(()) } +/// Decodes a whole compressed buffer held in memory. +pub fn decompress_slice( + bytes: &[u8], + method: CompressionMethod, +) -> Result, CompressionError> { + #[cfg(feature = "zstd-rust")] + if method == CompressionMethod::ZstdRust { + use structured_zstd::decoding::{FrameContentSize, FrameDecoder, read_frame_content_size}; + + fn zstd_rust_error( + err: impl Into>, + ) -> CompressionError { + CompressionError::ZstdRustError(io::Error::other(err)) + } + + // Frame_Content_Size is optional (RFC 8878 3.1.1.1.2): with it the + // frame decodes in one call straight into an exactly sized buffer, + // without it only the streaming decoder can tell where the data ends. + if let FrameContentSize::Known(size) = + read_frame_content_size(bytes).map_err(zstd_rust_error)? + { + let size = usize::try_from(size).map_err(zstd_rust_error)?; + let mut out = vec![0; size]; + let written = FrameDecoder::new() + .decode_all(bytes, &mut out) + .map_err(zstd_rust_error)?; + if written != size { + return Err(CompressionError::ZstdRustError(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("frame declares {size} bytes but holds {written}"), + ))); + } + return Ok(out); + } + } + + let mut out = Vec::new(); + apply_decompression(bytes, &mut out, method)?; + Ok(out) +} + #[cfg(test)] mod tests; diff --git a/compress/src/tests.rs b/compress/src/tests.rs index 1e94290..ceb9dc2 100644 --- a/compress/src/tests.rs +++ b/compress/src/tests.rs @@ -13,12 +13,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! `Zstd` and `ZstdRust` share the Zstandard frame format, so each one -//! decodes what the other encoded. +#![cfg(feature = "zstd-rust")] -#![cfg(all(feature = "zstd", feature = "zstd-rust"))] - -use super::{CompressionMethod, apply_compression, apply_decompression}; +use super::{CompressionMethod, apply_compression, decompress_slice}; fn sample() -> Vec { (0..200_000u32) @@ -26,22 +23,122 @@ fn sample() -> Vec { .collect() } -fn round_trip(encode: CompressionMethod, decode: CompressionMethod) { - let data = sample(); +fn encode(data: &[u8], method: CompressionMethod) -> Vec { let mut frame = Vec::new(); - apply_compression(&mut &data[..], &mut frame, encode).unwrap(); + apply_compression(&mut &data[..], &mut frame, method).unwrap(); + frame +} + +/// `Zstd` and `ZstdRust` share the Zstandard frame format, so each one +/// decodes what the other encoded, through both decoding entry points. +#[cfg(feature = "zstd")] +fn round_trip(encode_with: CompressionMethod, decode_with: CompressionMethod) { + use super::apply_decompression; + + let data = sample(); + let frame = encode(&data, encode_with); let mut decoded = Vec::new(); - apply_decompression(&frame[..], &mut decoded, decode).unwrap(); + apply_decompression(&frame[..], &mut decoded, decode_with).unwrap(); assert_eq!(decoded, data); + + assert_eq!(decompress_slice(&frame, decode_with).unwrap(), data); } +#[cfg(feature = "zstd")] #[test] fn a_zstd_frame_decodes_with_zstd_rust() { round_trip(CompressionMethod::Zstd, CompressionMethod::ZstdRust); } +#[cfg(feature = "zstd")] #[test] fn a_zstd_rust_frame_decodes_with_zstd() { round_trip(CompressionMethod::ZstdRust, CompressionMethod::Zstd); } + +/// The one-shot decode sizes its output from `Frame_Content_Size`, so a +/// `ZstdRust` frame must declare it. +#[test] +fn a_zstd_rust_frame_declares_its_content_size() { + use structured_zstd::decoding::{FrameContentSize, read_frame_content_size}; + + let data = sample(); + let frame = encode(&data, CompressionMethod::ZstdRust); + assert_eq!( + read_frame_content_size(&frame).unwrap(), + FrameContentSize::Known(data.len() as u64) + ); + assert_eq!( + decompress_slice(&frame, CompressionMethod::ZstdRust).unwrap(), + data + ); +} + +/// Empty input still makes a frame that declares its (zero) size and decodes +/// back to nothing. +#[test] +fn an_empty_zstd_rust_frame_round_trips() { + let frame = encode(&[], CompressionMethod::ZstdRust); + assert!( + decompress_slice(&frame, CompressionMethod::ZstdRust) + .unwrap() + .is_empty() + ); +} + +/// `Frame_Content_Size` is optional (RFC 8878 3.1.1.1.2); a frame without it +/// still decodes, through the streaming decoder. +#[test] +fn a_zstd_rust_frame_without_content_size_decodes() { + use std::io::Write; + use structured_zstd::decoding::{FrameContentSize, read_frame_content_size}; + use structured_zstd::encoding::{CompressionLevel, StreamingEncoder}; + + let data = sample(); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Default); + encoder.write_all(&data).unwrap(); + let frame = encoder.finish().unwrap(); + assert_eq!( + read_frame_content_size(&frame).unwrap(), + FrameContentSize::Unknown + ); + + assert_eq!( + decompress_slice(&frame, CompressionMethod::ZstdRust).unwrap(), + data + ); +} + +/// A frame whose blocks hold less than its header declares is reported as an +/// error, not returned with a zero-filled tail. +#[test] +fn a_zstd_rust_frame_shorter_than_declared_is_an_error() { + use structured_zstd::decoding::{FrameContentSize, read_frame_content_size}; + + let data = sample(); + let mut frame = encode(&data, CompressionMethod::ZstdRust); + + // RFC 8878 3.1.1.1: magic (4), descriptor (1), then the window descriptor + // unless Single_Segment_flag is set, the dictionary ID, and the + // little-endian Frame_Content_Size field. + let descriptor = frame[4]; + let single_segment = descriptor & 0x20 != 0; + let dict_id_len = [0, 1, 2, 4][usize::from(descriptor & 0x03)]; + let fcs_len = match descriptor >> 6 { + 0 => usize::from(single_segment), + 1 => 2, + 2 => 4, + _ => 8, + }; + assert_eq!(fcs_len, 4, "the sample needs a 4-byte FCS field"); + let at = 5 + usize::from(!single_segment) + dict_id_len; + let declared = u32::from_le_bytes(frame[at..at + 4].try_into().unwrap()); + frame[at..at + 4].copy_from_slice(&(declared + 1).to_le_bytes()); + assert_eq!( + read_frame_content_size(&frame).unwrap(), + FrameContentSize::Known(data.len() as u64 + 1) + ); + + assert!(decompress_slice(&frame, CompressionMethod::ZstdRust).is_err()); +} diff --git a/src/lib.rs b/src/lib.rs index cd192b9..a4d0b77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,7 @@ /// The low-level macros used by this crate. pub use include_flate_codegen as codegen; -use include_flate_compress::apply_decompression; +use include_flate_compress::decompress_slice; use std::string::FromUtf8Error; pub use include_flate_compress::CompressionMethod; @@ -184,18 +184,11 @@ impl IFlate { #[doc(hidden)] #[allow(private_interfaces)] pub fn decode(bytes: &[u8], algo: Option) -> Vec { - use std::io::Cursor; - let algo = algo.unwrap_or_default(); - let mut source = Cursor::new(bytes); - let mut ret = Vec::new(); - - match apply_decompression(&mut source, &mut ret, algo) { - Ok(_) => {} + match decompress_slice(bytes, algo) { + Ok(ret) => ret, Err(err) => panic!("Compiled `{:?}` buffer was corrupted: {:?}", algo, err), } - - ret } #[doc(hidden)] diff --git a/test_util.rs b/test_util.rs index 37cc336..0ea1d27 100644 --- a/test_util.rs +++ b/test_util.rs @@ -18,6 +18,8 @@ use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::str::from_utf8; +#[cfg(all(feature = "zstd", feature = "zstd-rust"))] +use include_flate_compress::decompress_slice; use include_flate_compress::{CompressionMethod, apply_compression, apply_decompression}; pub fn get_file_path>(relative_from: Option<&Path>, path: P) -> PathBuf { @@ -66,6 +68,7 @@ pub fn verify_cross>(name: P, encode: CompressionMethod, decode: let mut decompressed = Vec::new(); apply_decompression(&compressed[..], &mut decompressed, decode).unwrap(); assert_eq!(decompressed, original); + assert_eq!(decompress_slice(&compressed, decode).unwrap(), original); } pub fn verify>(name: P, data: &[u8]) { From acb9f72460a9e4f262149833645fb08a7a05d6a3 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 7 Oct 2026 19:17:03 +0300 Subject: [PATCH 6/9] Decode every zstd_rust frame and bound declared sizes - FlateEncoder::finish is public, so callers of CompressionMethod::encoder can end the stream; the zstd_rust encoder writes its whole frame there. - decode_zstd_rust walks the frames first. When each declares Frame_Content_Size it decodes them all in one call into a buffer of their total size, otherwise it uses the library's read_to_end, which continues through following frames; skippable frames are skipped either way. The zstd_rust Read decoder decodes the stream this way on its first read, so a concatenated stream is no longer cut after its first frame. - A declared size larger than the frame can hold (RFC 8878 3.1.1.2: at least 3 bytes on disk and at most 128 KiB out per block) is an error instead of a capacity-overflow panic or an allocation of that size. - Regression tests: the public finish path, an impossible declared size, concatenated frames with and without declared sizes, leading and interleaved skippable frames, trailing bytes that are not a frame. --- compress/src/lib.rs | 140 +++++++++++++++++++++++++++++++----------- compress/src/tests.rs | 138 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 237 insertions(+), 41 deletions(-) diff --git a/compress/src/lib.rs b/compress/src/lib.rs index 1fad765..06ec175 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -30,11 +30,102 @@ use zstd::Decoder as ZstdDecoder; #[cfg(feature = "zstd")] use zstd::Encoder as ZstdEncoder; -// The pure-Rust decoder keeps its state inline (the C one holds a pointer to -// it), so it is boxed to keep the enum below the size of its other variants. +/// Decodes the whole stream on the first read, every frame of it (RFC 8878 +/// 3), and hands the result out from then on. #[cfg(feature = "zstd-rust")] -type ZstdRustDecoder = - Box>; +pub struct ZstdRustDecoder { + source: Option, + decoded: io::Cursor>, +} + +#[cfg(feature = "zstd-rust")] +impl Read for ZstdRustDecoder { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if let Some(mut source) = self.source.take() { + let mut input = Vec::new(); + source.read_to_end(&mut input)?; + self.decoded = io::Cursor::new(decode_zstd_rust(&input)?); + } + self.decoded.read(buf) + } +} + +/// Decodes a Zstandard stream held in memory: every frame, skippable frames +/// skipped (RFC 8878 3). +#[cfg(feature = "zstd-rust")] +fn decode_zstd_rust(bytes: &[u8]) -> io::Result> { + use structured_zstd::decoding::errors::ReadFrameHeaderError; + use structured_zstd::decoding::{ + FrameContentSize, FrameDecoder, StreamingDecoder, find_frame_compressed_size, + read_frame_content_size, + }; + + fn invalid(err: impl fmt::Debug) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, format!("{err:?}")) + } + + // Walk the frames first: when every one declares Frame_Content_Size + // (optional, RFC 8878 3.1.1.1.2), the stream decodes in one call straight + // into a buffer of exactly their total size. + let mut total = 0u64; + let mut all_declared = true; + let mut first_data_frame = None; + let mut offset = 0; + while offset < bytes.len() { + let frame = &bytes[offset..]; + let len = find_frame_compressed_size(frame).map_err(invalid)?; + match read_frame_content_size(frame) { + Ok(size) => { + first_data_frame.get_or_insert(offset); + if let FrameContentSize::Known(size) = size { + // RFC 8878 3.1.1.2: every block takes at least its 3-byte + // header on disk and regenerates at most 128 KiB, so a + // frame declaring more than that cannot be honest. + let max = (len as u64 / 3) * (128 * 1024); + if size > max { + return Err(invalid(format!( + "frame declares {size} bytes but its {len} bytes hold at most {max}" + ))); + } + // Each term is bounded by its frame's length as above, so + // the sum stays far below u64::MAX. + total += size; + } else { + all_declared = false; + } + } + Err(ReadFrameHeaderError::SkipFrame { .. }) => {} + Err(err) => return Err(invalid(err)), + } + offset += len; + } + + let Some(first_data_frame) = first_data_frame else { + return Ok(Vec::new()); + }; + if all_declared { + let size = usize::try_from(total).map_err(invalid)?; + let mut out = vec![0; size]; + let written = FrameDecoder::new() + .decode_all(bytes, &mut out) + .map_err(invalid)?; + if written != size { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("frames declare {size} bytes but hold {written}"), + )); + } + return Ok(out); + } + + // `read_to_end` goes on through the following frames, unlike `read`; it + // has to start at a frame that is not skippable. + let mut out = Vec::new(); + StreamingDecoder::new(&bytes[first_data_frame..]) + .map_err(invalid)? + .read_to_end(&mut out)?; + Ok(out) +} /// Collects the whole input and encodes it as one frame on finish, so the /// frame declares `Frame_Content_Size` and `decompress_slice` can decode it @@ -190,7 +281,9 @@ impl Write for FlateEncoder { } impl FlateEncoder { - fn finish_encode(self) -> Result { + /// Ends the stream and returns the writer. Every method needs this to + /// write its final bytes; `ZstdRust` writes the whole frame here. + pub fn finish(self) -> Result { match self { #[cfg(feature = "deflate")] FlateEncoder::Deflate(encoder) => encoder @@ -235,9 +328,10 @@ impl FlateDecoder { Ok(FlateDecoder::Zstd(decoder)) } #[cfg(feature = "zstd-rust")] - CompressionMethod::ZstdRust => structured_zstd::decoding::StreamingDecoder::new(read) - .map(|decoder| FlateDecoder::ZstdRust(Box::new(decoder))) - .map_err(|err| CompressionError::ZstdRustError(io::Error::other(err))), + CompressionMethod::ZstdRust => Ok(FlateDecoder::ZstdRust(ZstdRustDecoder { + source: Some(read), + decoded: io::Cursor::new(Vec::new()), + })), } } } @@ -266,7 +360,7 @@ where { let mut encoder = method.encoder(writer)?; io::copy(reader, &mut encoder)?; - encoder.finish_encode().map(|_| ()) + encoder.finish().map(|_| ()) } pub fn apply_decompression( @@ -286,33 +380,7 @@ pub fn decompress_slice( ) -> Result, CompressionError> { #[cfg(feature = "zstd-rust")] if method == CompressionMethod::ZstdRust { - use structured_zstd::decoding::{FrameContentSize, FrameDecoder, read_frame_content_size}; - - fn zstd_rust_error( - err: impl Into>, - ) -> CompressionError { - CompressionError::ZstdRustError(io::Error::other(err)) - } - - // Frame_Content_Size is optional (RFC 8878 3.1.1.1.2): with it the - // frame decodes in one call straight into an exactly sized buffer, - // without it only the streaming decoder can tell where the data ends. - if let FrameContentSize::Known(size) = - read_frame_content_size(bytes).map_err(zstd_rust_error)? - { - let size = usize::try_from(size).map_err(zstd_rust_error)?; - let mut out = vec![0; size]; - let written = FrameDecoder::new() - .decode_all(bytes, &mut out) - .map_err(zstd_rust_error)?; - if written != size { - return Err(CompressionError::ZstdRustError(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!("frame declares {size} bytes but holds {written}"), - ))); - } - return Ok(out); - } + return decode_zstd_rust(bytes).map_err(CompressionError::ZstdRustError); } let mut out = Vec::new(); diff --git a/compress/src/tests.rs b/compress/src/tests.rs index ceb9dc2..bfbcd2c 100644 --- a/compress/src/tests.rs +++ b/compress/src/tests.rs @@ -91,14 +91,10 @@ fn an_empty_zstd_rust_frame_round_trips() { /// still decodes, through the streaming decoder. #[test] fn a_zstd_rust_frame_without_content_size_decodes() { - use std::io::Write; use structured_zstd::decoding::{FrameContentSize, read_frame_content_size}; - use structured_zstd::encoding::{CompressionLevel, StreamingEncoder}; let data = sample(); - let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Default); - encoder.write_all(&data).unwrap(); - let frame = encoder.finish().unwrap(); + let frame = frame_without_content_size(&data); assert_eq!( read_frame_content_size(&frame).unwrap(), FrameContentSize::Unknown @@ -142,3 +138,135 @@ fn a_zstd_rust_frame_shorter_than_declared_is_an_error() { assert!(decompress_slice(&frame, CompressionMethod::ZstdRust).is_err()); } + +/// A frame encoded without `Frame_Content_Size`, as a streaming encoder that +/// does not know the input length writes it. +fn frame_without_content_size(data: &[u8]) -> Vec { + use std::io::Write; + use structured_zstd::encoding::{CompressionLevel, StreamingEncoder}; + + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Default); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + +/// RFC 8878 3.1.2: a skippable frame carrying `payload`. +fn skippable_frame(payload: &[u8]) -> Vec { + let mut frame = 0x184D_2A50u32.to_le_bytes().to_vec(); + frame.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + frame.extend_from_slice(payload); + frame +} + +fn decode_streaming(bytes: &[u8]) -> Vec { + let mut out = Vec::new(); + super::apply_decompression(bytes, &mut out, CompressionMethod::ZstdRust).unwrap(); + out +} + +/// The encoder handed out by `CompressionMethod::encoder` can be finished +/// by its caller, which is what writes the frame. +#[test] +fn the_public_zstd_rust_encoder_can_be_finished() { + use std::io::Write; + + let data = sample(); + let mut encoder = CompressionMethod::ZstdRust.encoder(Vec::new()).unwrap(); + encoder.write_all(&data).unwrap(); + let frame = encoder.finish().unwrap(); + assert_eq!( + decompress_slice(&frame, CompressionMethod::ZstdRust).unwrap(), + data + ); +} + +/// A tiny frame declaring a content size no frame of its length can hold is +/// an error, not a capacity-overflow panic or an allocation of that size. +#[test] +fn a_zstd_rust_frame_declaring_an_impossible_size_is_an_error() { + // RFC 8878 3.1.1: magic, descriptor with an 8-byte Frame_Content_Size and + // Single_Segment_flag, the size, then one empty last raw block. + let mut frame = 0xFD2F_B528u32.to_le_bytes().to_vec(); + frame.push(0xE0); + frame.extend_from_slice(&u64::MAX.to_le_bytes()); + frame.extend_from_slice(&[0x01, 0x00, 0x00]); + + assert!(decompress_slice(&frame, CompressionMethod::ZstdRust).is_err()); +} + +/// RFC 8878 3: a stream is a sequence of frames. Frames that declare their +/// size decode in one call, through every frame, on both entry points. +#[test] +fn concatenated_zstd_rust_frames_decode_completely() { + let data = sample(); + let (first, second) = data.split_at(data.len() / 3); + let mut stream = encode(first, CompressionMethod::ZstdRust); + stream.extend_from_slice(&encode(second, CompressionMethod::ZstdRust)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// Frames without a declared size, with a skippable frame between them, +/// decode completely; the skippable frame contributes nothing. +#[test] +fn concatenated_frames_without_content_size_decode_completely() { + let data = sample(); + let (first, second) = data.split_at(data.len() / 3); + let mut stream = frame_without_content_size(first); + stream.extend_from_slice(&skippable_frame(b"meta")); + stream.extend_from_slice(&frame_without_content_size(second)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// One frame with a declared size and one without still decode completely. +#[test] +fn frames_with_and_without_content_size_decode_completely() { + let data = sample(); + let (first, second) = data.split_at(data.len() / 2); + let mut stream = encode(first, CompressionMethod::ZstdRust); + stream.extend_from_slice(&frame_without_content_size(second)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// A skippable frame ahead of the data is skipped by both entry points. +#[test] +fn a_leading_skippable_frame_is_skipped() { + let data = sample(); + let mut stream = skippable_frame(&[7; 16]); + stream.extend_from_slice(&encode(&data, CompressionMethod::ZstdRust)); + + assert_eq!( + decompress_slice(&stream, CompressionMethod::ZstdRust).unwrap(), + data + ); + assert_eq!(decode_streaming(&stream), data); +} + +/// Bytes after the last frame that do not form a frame are an error, not +/// silently dropped. +#[test] +fn trailing_garbage_after_a_zstd_rust_frame_is_an_error() { + let data = sample(); + let mut stream = encode(&data, CompressionMethod::ZstdRust); + stream.extend_from_slice(b"garbage"); + + assert!(decompress_slice(&stream, CompressionMethod::ZstdRust).is_err()); + let mut out = Vec::new(); + assert!( + super::apply_decompression(&stream[..], &mut out, CompressionMethod::ZstdRust).is_err() + ); +} From 4af3a09a72645308113d9456dd3738758ce0745a Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Wed, 7 Oct 2026 19:27:29 +0300 Subject: [PATCH 7/9] Reject empty input as a zstd_rust stream RFC 8878 3: compressed data is one or more frames, so an empty input (a fully truncated embedded frame) is InvalidData rather than an empty successful decode. Regression test: empty_input_is_not_a_zstd_rust_stream. --- compress/src/lib.rs | 5 +++++ compress/src/tests.rs | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/compress/src/lib.rs b/compress/src/lib.rs index 06ec175..1b35733 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -64,6 +64,11 @@ fn decode_zstd_rust(bytes: &[u8]) -> io::Result> { io::Error::new(io::ErrorKind::InvalidData, format!("{err:?}")) } + // RFC 8878 3: compressed data is one or more frames. + if bytes.is_empty() { + return Err(invalid("empty input holds no Zstandard frame")); + } + // Walk the frames first: when every one declares Frame_Content_Size // (optional, RFC 8878 3.1.1.1.2), the stream decodes in one call straight // into a buffer of exactly their total size. diff --git a/compress/src/tests.rs b/compress/src/tests.rs index bfbcd2c..0eb0058 100644 --- a/compress/src/tests.rs +++ b/compress/src/tests.rs @@ -242,6 +242,13 @@ fn frames_with_and_without_content_size_decode_completely() { assert_eq!(decode_streaming(&stream), data); } +/// RFC 8878 3: compressed data is one or more frames, so an empty input is +/// not a stream (a fully truncated frame) and must not decode to nothing. +#[test] +fn empty_input_is_not_a_zstd_rust_stream() { + assert!(decompress_slice(&[], CompressionMethod::ZstdRust).is_err()); +} + /// A skippable frame ahead of the data is skipped by both entry points. #[test] fn a_leading_skippable_frame_is_skipped() { From 0b42518ec85b1327bd978d461fdf93c9ccae36eb Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Thu, 8 Oct 2026 16:33:24 +0300 Subject: [PATCH 8/9] docs(compress): scope decompress_slice to trusted data For ZstdRust the output is allocated at the size the frames declare, bounded only by what their length could hold, so a crafted input can request far more memory than it decodes to. decompress_slice is the path for data this crate encoded (what flate! embeds); untrusted input belongs on the streaming decoder. --- compress/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compress/src/lib.rs b/compress/src/lib.rs index 1b35733..3fd2afa 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -379,6 +379,12 @@ pub fn apply_decompression( } /// Decodes a whole compressed buffer held in memory. +/// +/// Meant for data this crate encoded, such as what `flate!` embeds. For +/// `ZstdRust` the output is allocated up front at the size the frames declare, +/// bounded only by what their length could hold, so a crafted input can ask +/// for far more memory than it decodes to; decode untrusted input through +/// [`CompressionMethod::decoder`] instead. pub fn decompress_slice( bytes: &[u8], method: CompressionMethod, From 560e0053c680a22bc579fe2bc450bee1f79b47ed Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Thu, 8 Oct 2026 16:43:39 +0300 Subject: [PATCH 9/9] fix(compress): stream zstd_rust through StreamingDecoder 0.0.59 structured-zstd 0.0.59 makes StreamingDecoder::read decode every frame of a stream (RFC 8878 3), skip skippable frames wherever they stand, including first, and hand out a finished frame's bytes before reading the next header. The zstd_rust Read decoder goes back to a plain StreamingDecoder, so it streams again instead of waiting for the end of its source and holding the whole input and output in memory, and keeps its source after a WouldBlock. decompress_slice keeps the one-call path for frames that declare their size and falls back to read_to_end over the whole stream otherwise. Tests: small reads across concatenated and skippable frames, a source that answers WouldBlock after a whole frame (fails on the previous read-to-end decoder), skippable frames alone, empty input on both entry points. --- Cargo.toml | 2 +- compress/src/lib.rs | 38 +++++------------------ compress/src/tests.rs | 72 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5ed1f32..a313b30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ license = "Apache-2.0" [workspace.dependencies] libflate = "2.3.0" zstd = "0.14.0" -structured-zstd = "0.0.56" +structured-zstd = "0.0.59" [package] name = "include-flate" diff --git a/compress/src/lib.rs b/compress/src/lib.rs index 3fd2afa..ae9e52c 100644 --- a/compress/src/lib.rs +++ b/compress/src/lib.rs @@ -30,25 +30,11 @@ use zstd::Decoder as ZstdDecoder; #[cfg(feature = "zstd")] use zstd::Encoder as ZstdEncoder; -/// Decodes the whole stream on the first read, every frame of it (RFC 8878 -/// 3), and hands the result out from then on. +// The pure-Rust decoder keeps its state inline (the C one holds a pointer to +// it), so it is boxed to keep the enum below the size of its other variants. #[cfg(feature = "zstd-rust")] -pub struct ZstdRustDecoder { - source: Option, - decoded: io::Cursor>, -} - -#[cfg(feature = "zstd-rust")] -impl Read for ZstdRustDecoder { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if let Some(mut source) = self.source.take() { - let mut input = Vec::new(); - source.read_to_end(&mut input)?; - self.decoded = io::Cursor::new(decode_zstd_rust(&input)?); - } - self.decoded.read(buf) - } -} +type ZstdRustDecoder = + Box>; /// Decodes a Zstandard stream held in memory: every frame, skippable frames /// skipped (RFC 8878 3). @@ -74,14 +60,12 @@ fn decode_zstd_rust(bytes: &[u8]) -> io::Result> { // into a buffer of exactly their total size. let mut total = 0u64; let mut all_declared = true; - let mut first_data_frame = None; let mut offset = 0; while offset < bytes.len() { let frame = &bytes[offset..]; let len = find_frame_compressed_size(frame).map_err(invalid)?; match read_frame_content_size(frame) { Ok(size) => { - first_data_frame.get_or_insert(offset); if let FrameContentSize::Known(size) = size { // RFC 8878 3.1.1.2: every block takes at least its 3-byte // header on disk and regenerates at most 128 KiB, so a @@ -105,9 +89,6 @@ fn decode_zstd_rust(bytes: &[u8]) -> io::Result> { offset += len; } - let Some(first_data_frame) = first_data_frame else { - return Ok(Vec::new()); - }; if all_declared { let size = usize::try_from(total).map_err(invalid)?; let mut out = vec![0; size]; @@ -123,10 +104,8 @@ fn decode_zstd_rust(bytes: &[u8]) -> io::Result> { return Ok(out); } - // `read_to_end` goes on through the following frames, unlike `read`; it - // has to start at a frame that is not skippable. let mut out = Vec::new(); - StreamingDecoder::new(&bytes[first_data_frame..]) + StreamingDecoder::new(bytes) .map_err(invalid)? .read_to_end(&mut out)?; Ok(out) @@ -333,10 +312,9 @@ impl FlateDecoder { Ok(FlateDecoder::Zstd(decoder)) } #[cfg(feature = "zstd-rust")] - CompressionMethod::ZstdRust => Ok(FlateDecoder::ZstdRust(ZstdRustDecoder { - source: Some(read), - decoded: io::Cursor::new(Vec::new()), - })), + CompressionMethod::ZstdRust => structured_zstd::decoding::StreamingDecoder::new(read) + .map(|decoder| FlateDecoder::ZstdRust(Box::new(decoder))) + .map_err(|err| CompressionError::ZstdRustError(io::Error::other(err))), } } } diff --git a/compress/src/tests.rs b/compress/src/tests.rs index 0eb0058..2e6aaa6 100644 --- a/compress/src/tests.rs +++ b/compress/src/tests.rs @@ -247,6 +247,78 @@ fn frames_with_and_without_content_size_decode_completely() { #[test] fn empty_input_is_not_a_zstd_rust_stream() { assert!(decompress_slice(&[], CompressionMethod::ZstdRust).is_err()); + let mut out = Vec::new(); + assert!(super::apply_decompression(&[][..], &mut out, CompressionMethod::ZstdRust).is_err()); +} + +/// A stream of skippable frames only holds no data: both entry points decode +/// it to nothing. +#[test] +fn skippable_frames_alone_decode_to_nothing() { + let mut stream = skippable_frame(b"one"); + stream.extend_from_slice(&skippable_frame(b"two")); + + assert!( + decompress_slice(&stream, CompressionMethod::ZstdRust) + .unwrap() + .is_empty() + ); + assert!(decode_streaming(&stream).is_empty()); +} + +/// The `Read` decoder hands concatenated frames out through reads smaller +/// than a block, across the frame boundary. +#[test] +fn the_zstd_rust_decoder_reads_concatenated_frames_in_small_pieces() { + use std::io::Read; + + let data = sample(); + let (first, second) = data.split_at(data.len() / 3); + let mut stream = encode(first, CompressionMethod::ZstdRust); + stream.extend_from_slice(&skippable_frame(b"meta")); + stream.extend_from_slice(&frame_without_content_size(second)); + + let mut decoder = CompressionMethod::ZstdRust.decoder(&stream[..]).unwrap(); + let mut out = Vec::new(); + let mut piece = [0u8; 7]; + loop { + let n = decoder.read(&mut piece).unwrap(); + if n == 0 { + break; + } + out.extend_from_slice(&piece[..n]); + } + assert_eq!(out, data); +} + +/// The `Read` decoder stays streaming: once a whole frame has arrived from a +/// source that stays open (here it answers `WouldBlock` after the frame), its +/// bytes are handed out without waiting for the source to end. +#[test] +fn the_zstd_rust_decoder_does_not_wait_for_the_source_to_end() { + use std::io::{self, Read}; + + struct OpenSource<'a>(&'a [u8]); + impl Read for OpenSource<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.0.is_empty() { + return Err(io::ErrorKind::WouldBlock.into()); + } + let n = buf.len().min(self.0.len()); + buf[..n].copy_from_slice(&self.0[..n]); + self.0 = &self.0[n..]; + Ok(n) + } + } + + let data = sample(); + let frame = encode(&data, CompressionMethod::ZstdRust); + let mut decoder = CompressionMethod::ZstdRust + .decoder(OpenSource(&frame)) + .unwrap(); + let mut out = vec![0; data.len()]; + decoder.read_exact(&mut out).unwrap(); + assert_eq!(out, data); } /// A skippable frame ahead of the data is skipped by both entry points.